Tutorial 6: Introduction to Convolutional Neural Networks (CNNs)

train_dataset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transforms.ToTensor())
# Take one image from the training set
image, label = train_dataset[0]

print("Original shape:", image.shape)  # Should be [1, 28, 28]

# Flatten the image to a 1D vector of length 784
flattened = image.view(-1)
print("Flattened shape:", flattened.shape)  # Should be [784]

# Optionally, show the first 20 values
print("First 20 pixel values:", flattened[:20])

import matplotlib.pyplot as plt

# Display the original image
plt.figure(figsize=(8, 2))

plt.subplot(1, 2, 1)
plt.imshow(image.squeeze(), cmap='gray')
plt.title(f"Label: {label}")
plt.axis('off')

# Display the flattened image as a row vector
plt.subplot(1, 2, 2)
plt.imshow(flattened.unsqueeze(0), cmap='gray', aspect='auto')
plt.title("Flattened (1x784)")
plt.xlabel("Pixel Index")
plt.yticks([])
plt.tight_layout()
plt.show()

import torchvision.datasets as datasets
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import ipywidgets as widgets
from IPython.display import display, clear_output
import random

# 1. Download and load the MNIST dataset
print("Loading dataset... (this may take a moment if downloading for the first time)")
dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transforms.ToTensor())
clear_output() # Clears the loading text once done

# 2. Global variables to store the current state
current_image = None
current_label = None

# 3. Create the UI widgets
out = widgets.Output()
guess_input = widgets.BoundedIntText(value=0, min=0, max=9, step=1, description='Guess:', layout=widgets.Layout(width='150px'))
submit_btn = widgets.Button(description='Submit Guess', button_style='success')
next_btn = widgets.Button(description='Load New Image', button_style='info')
feedback_label = widgets.HTML(value="<b>Click 'Load New Image' to begin!</b>")

# 4. Define button actions (callbacks)
def load_new_image(b):
    global current_image, current_label
    
    # Pick a random image
    idx = random.randint(0, len(dataset) - 1)
    current_image, current_label = dataset[idx]
    
    # Update UI for a new round
    feedback_label.value = "<b>Examine the 1D vector below. Enter your guess (0-9) and click Submit!</b>"
    
    with out:
        clear_output(wait=True)
        flattened = current_image.view(-1).unsqueeze(0)
        
        plt.figure(figsize=(10, 2))
        plt.imshow(flattened, cmap='gray', aspect='auto')
        plt.title("Mystery Flattened Image (1 x 784)")
        plt.xlabel("Pixel Index (0 to 783)")
        plt.yticks([])
        plt.show()

def submit_guess(b):
    global current_image, current_label
    if current_image is None:
        return
        
    guess = guess_input.value
    
    # Provide feedback
    if guess == current_label:
        feedback_label.value = f"<b style='color:green;'>Spot on!</b> You successfully decoded the matrix. It is a <b>{current_label}</b>."
    else:
        feedback_label.value = f"<b style='color:red;'>Not quite.</b> You guessed {guess}, but it was actually a <b>{current_label}</b>. This is why MLPs struggle!"
        
    # Reveal the original 2D image alongside the 1D vector
    with out:
        clear_output(wait=True)
        flattened = current_image.view(-1).unsqueeze(0)
        
        fig, axes = plt.subplots(2, 1, figsize=(10, 5))
        
        # Display Flattened
        axes[0].imshow(flattened, cmap='gray', aspect='auto')
        axes[0].set_title("What the MLP saw (Flattened 1D Vector)")
        axes[0].set_yticks([])
        
        # Display Original
        axes[1].imshow(current_image.squeeze(), cmap='gray')
        axes[1].set_title(f"What the CNN sees (Preserved 2D Topology)")
        axes[1].axis('off')
        
        plt.tight_layout()
        plt.show()

# 5. Connect buttons to actions
next_btn.on_click(load_new_image)
submit_btn.on_click(submit_guess)

# 6. Layout and display the UI
controls = widgets.HBox([guess_input, submit_btn, next_btn])
display(feedback_label, controls, out)

import torch
import torchvision.datasets as datasets
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import ipywidgets as widgets
from IPython.display import display, clear_output
import random
import math

# 1. Load MNIST Dataset
print("Loading MNIST...")
dataset_mnist = datasets.MNIST(root='./data', train=True, download=True, transform=transforms.ToTensor())
clear_output()

# --- Helper Function for Spaced-Out Grayscale Patches ---
def get_spaced_mnist_patches(img_tensor, num_patches=3, patch_size=6):
    img_2d = img_tensor.squeeze()
    h, w = img_2d.shape
    patch_size = min(patch_size, h)
    
    if patch_size == h:
        return [img_2d.numpy()]*num_patches, [(0,0)]*num_patches, patch_size

    # 1. Calculate variance for all patches
    scored_patches = []
    for r in range(h - patch_size):
        for col in range(w - patch_size):
            patch = img_2d[r:r+patch_size, col:col+patch_size]
            variance = torch.var(patch).item() 
            # Ignore completely blank/black areas to speed things up
            if variance > 0.01: 
                scored_patches.append((variance, r, col))
            
    # Sort by variance (highest complexity first)
    scored_patches.sort(reverse=True, key=lambda x: x[0])
    
    # 2. Select patches with a distance constraint
    chosen = []
    # Require patches to be separated by at least 75% of the patch width, minimum 5 pixels
    min_dist = max(patch_size * 0.75, 5.0) 
    
    for cand_var, cand_r, cand_c in scored_patches:
        too_close = False
        for _, chosen_r, chosen_c in chosen:
            # Euclidean distance formula
            dist = math.sqrt((cand_r - chosen_r)**2 + (cand_c - chosen_c)**2)
            if dist < min_dist:
                too_close = True
                break
                
        if not too_close:
            chosen.append((cand_var, cand_r, cand_c))
            
        if len(chosen) == num_patches:
            break
            
    # 3. Fallback (If the digit is too small to find 3 widely spaced patches)
    if len(chosen) < num_patches and len(scored_patches) > 0:
        top_pool = scored_patches[:max(1, len(scored_patches)//10)]
        for cand_var, cand_r, cand_c in top_pool:
            if (cand_var, cand_r, cand_c) not in chosen:
                chosen.append((cand_var, cand_r, cand_c))
            if len(chosen) == num_patches:
                break

    # Shuffle so the most complex patch isn't always the first one displayed
    random.shuffle(chosen)

    extracted_patches = []
    patch_coords = []
    
    for _, r, col in chosen:
        patch = img_2d[r:r+patch_size, col:col+patch_size]
        extracted_patches.append(patch.numpy())
        patch_coords.append((col, r))
        
    return extracted_patches, patch_coords, patch_size

# --- Game State & UI ---
current_image_mnist = None
current_label_mnist = None
current_coords_mnist = None
current_patch_size_mnist = None

out_mnist = widgets.Output()

patch_size_slider = widgets.IntSlider(
    value=6, min=2, max=28, step=2, 
    description='Patch Size:', 
    continuous_update=False,
    layout=widgets.Layout(width='300px')
)

guess_dropdown = widgets.Dropdown(options=list(range(10)), description='Guess Digit:')
submit_btn_mnist = widgets.Button(description='Submit Guess', button_style='success')
next_btn_mnist = widgets.Button(description='Load New Image', button_style='info')
feedback_label_mnist = widgets.HTML(value="<b>Adjust the patch size, guess the digit (0-9), and click Submit!</b>")

def load_new_mnist_round(b=None):
    global current_image_mnist, current_label_mnist, current_coords_mnist, current_patch_size_mnist
    
    is_new_image = (b is next_btn_mnist or current_image_mnist is None)
    
    if is_new_image:
        idx = random.randint(0, len(dataset_mnist) - 1)
        current_image_mnist, current_label_mnist = dataset_mnist[idx]
        feedback_label_mnist.value = "<b>Look at these distinct structural features. What digit do they form?</b>"
    
    p_size = patch_size_slider.value
    patches_list, coords, final_p_size = get_spaced_mnist_patches(current_image_mnist, num_patches=3, patch_size=p_size)
    current_coords_mnist = coords
    current_patch_size_mnist = final_p_size

    with out_mnist:
        clear_output(wait=True)
        fig, axes = plt.subplots(1, 3, figsize=(9, 3))
        for i, ax in enumerate(axes):
            if i < len(patches_list):
                ax.imshow(patches_list[i], cmap='gray', interpolation='nearest', vmin=0, vmax=1)
                ax.set_title(f"Receptive Field {i+1}\n({final_p_size}x{final_p_size})")
                
                if final_p_size <= 14:
                    import numpy as np
                    ax.set_xticks(np.arange(-.5, final_p_size, 1), minor=True)
                    ax.set_yticks(np.arange(-.5, final_p_size, 1), minor=True)
                    ax.grid(which='minor', color='red', linestyle='-', linewidth=0.5, alpha=0.3)
                    ax.tick_params(which='minor', bottom=False, left=False)
                    
                ax.axis('off')
        plt.tight_layout()
        plt.show()

def submit_mnist_guess(b):
    global current_image_mnist, current_label_mnist, current_coords_mnist, current_patch_size_mnist
    if current_image_mnist is None: return
        
    guess = guess_dropdown.value
    
    if guess == current_label_mnist:
        feedback_label_mnist.value = f"<b style='color:green;'>Excellent!</b> You correctly synthesized the dispersed features to find the <b>{current_label_mnist}</b>."
    else:
        feedback_label_mnist.value = f"<b style='color:red;'>Not quite.</b> You guessed {guess}, but it was a <b>{current_label_mnist}</b>."
        
    with out_mnist:
        clear_output(wait=True)
        fig, ax = plt.subplots(figsize=(5, 5))
        ax.imshow(current_image_mnist.squeeze(), cmap='gray')
        ax.set_title(f"Original Image: {current_label_mnist}")
        
        if current_patch_size_mnist < 28:
            for (col, r) in current_coords_mnist:
                rect = patches.Rectangle((col - 0.5, r - 0.5), 
                                         current_patch_size_mnist, current_patch_size_mnist, 
                                         linewidth=2, edgecolor='red', facecolor='none')
                ax.add_patch(rect)
            
        ax.axis('off')
        plt.show()

next_btn_mnist.on_click(load_new_mnist_round)
submit_btn_mnist.on_click(submit_mnist_guess)
patch_size_slider.observe(lambda change: load_new_mnist_round(), names='value')

top_row = widgets.HBox([patch_size_slider, next_btn_mnist])
bottom_row = widgets.HBox([guess_dropdown, submit_btn_mnist])
ui_layout = widgets.VBox([top_row, bottom_row])

display(feedback_label_mnist, ui_layout, out_mnist)
load_new_mnist_round()

import numpy as np
import matplotlib.pyplot as plt

# --- Toy convolution demo ---

image = np.array([
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
], dtype=np.float32)

kernel = np.array([
    [-1, 0, 1],
    [-1, 0, 1],
    [-1, 0, 1],
], dtype=np.float32)

def convolve2d(img, kern):
    h, w = img.shape
    kh, kw = kern.shape
    out_h, out_w = h - kh + 1, w - kw + 1
    output = np.zeros((out_h, out_w))
    for i in range(out_h):
        for j in range(out_w):
            patch = img[i:i+kh, j:j+kw]
            output[i, j] = np.sum(patch * kern)
    return output

feature_map = convolve2d(image, kernel)

fig, axes = plt.subplots(1, 3, figsize=(16, 5))

# 1. Input Image with Pixel Values
axes[0].imshow(image, cmap='gray', vmin=0, vmax=1)
axes[0].set_title('Input Image (6×6)', fontsize=13, pad=15)
for i in range(image.shape[0]):
    for j in range(image.shape[1]):
        # Dynamic color: white text on dark pixels, black on light
        color = "white" if image[i, j] < 0.5 else "black"
        axes[0].text(j, i, f'{image[i,j]:.0f}', ha='center', va='center', 
                     fontsize=12, fontweight='bold', color=color)
axes[0].axis('off')

# 2. Filter / Kernel
axes[1].imshow(kernel, cmap='RdBu', vmin=-1, vmax=1)
axes[1].set_title('Filter / Kernel (3×3)', fontsize=13, pad=15)
for i in range(3):
    for j in range(3):
        axes[1].text(j, i, f'{kernel[i,j]:.0f}', ha='center', va='center', 
                     fontsize=14, fontweight='bold')
axes[1].axis('off')

# 3. Feature Map
axes[2].imshow(feature_map, cmap='hot')
axes[2].set_title('Feature Map (4×4)', fontsize=13, pad=15)
for i in range(feature_map.shape[0]):
    for j in range(feature_map.shape[1]):
        axes[2].text(j, i, f'{feature_map[i,j]:.0f}', ha='center', va='center', 
                     fontsize=14, fontweight='bold', color='cyan')
axes[2].axis('off')

plt.suptitle('Convolution in Action: Detecting a Vertical Edge', fontsize=16, fontweight='bold', y=1.05)
plt.tight_layout()
plt.show()

print('\n📝 Analysis:')
print(f'   The maximum value in the feature map is {np.max(feature_map):.0f}.')
print('   This occurs because the 3x3 kernel aligns perfectly with the 0-to-1 transition.')

import numpy as np
import plotly.graph_objects as go
from plotly.subplots import make_subplots

# --- 1. Define the Data ---
image = np.array([
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
    [0, 0, 0, 1, 1, 1],
], dtype=np.float32)

kernel = np.array([
    [-1, 0, 1],
    [-1, 0, 1],
    [-1, 0, 1],
], dtype=np.float32)

h, w = image.shape
kh, kw = kernel.shape
out_h, out_w = h - kh + 1, w - kw + 1

# Pre-calculate the full feature map
full_feature_map = np.zeros((out_h, out_w))
for i in range(out_h):
    for j in range(out_w):
        full_feature_map[i, j] = np.sum(image[i:i+kh, j:j+kw] * kernel)

# --- 2. Setup Figure and Base Traces ---
fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.1,
                    subplot_titles=("Input Image (6x6)", "Feature Map (Accumulating)"))

# Trace 0: Input Image Heatmap
fig.add_trace(go.Heatmap(z=image, colorscale='gray', showscale=False, zmin=0, zmax=1, hoverinfo='skip'), row=1, col=1)

# Trace 1: Input Image Text Overlay
img_y, img_x = np.mgrid[0:h, 0:w]
img_text = [f"{val:.0f}" for val in image.flatten()]
img_colors = ['white' if val < 0.5 else 'black' for val in image.flatten()]
fig.add_trace(go.Scatter(
    x=img_x.flatten(), y=img_y.flatten(), mode='text', text=img_text,
    textfont=dict(color=img_colors, size=14, family="Arial"), hoverinfo='skip'
), row=1, col=1)

# Trace 2: Feature Map Heatmap (Starts with NaNs so it's empty)
init_fm = np.full((out_h, out_w), np.nan)
fig.add_trace(go.Heatmap(z=init_fm, colorscale='hot', showscale=False, zmin=-3, zmax=3, hoverinfo='skip'), row=1, col=2)

# Trace 3: Feature Map Text Overlay (Starts empty)
fm_y, fm_x = np.mgrid[0:out_h, 0:out_w]
fig.add_trace(go.Scatter(
    x=fm_x.flatten(), y=fm_y.flatten(), mode='text', text=['']*(out_h*out_w),
    textfont=dict(color='cyan', size=14, family="Arial Black"), hoverinfo='skip'
), row=1, col=2)

# Trace 4: Sliding Window Bounding Box (Green Square)
fig.add_trace(go.Scatter(
    x=[-0.5, 2.5, 2.5, -0.5, -0.5], y=[-0.5, -0.5, 2.5, 2.5, -0.5],
    mode='lines', line=dict(color='lime', width=4), hoverinfo='skip'
), row=1, col=1)

# --- 3. Create Animation Frames ---
frames = []
steps = []

for step_idx in range(out_h * out_w):
    y_pos = step_idx // out_w
    x_pos = step_idx % out_w

    # Build the feature map state up to the current step
    curr_fm = np.full((out_h, out_w), np.nan)
    curr_text = [''] * (out_h * out_w)
    
    for k in range(step_idx + 1):
        r, c = k // out_w, k % out_w
        curr_fm[r, c] = full_feature_map[r, c]
        curr_text[k] = f"{full_feature_map[r, c]:.0f}"

    # Move the bounding box
    x_box = [x_pos - 0.5, x_pos + 2.5, x_pos + 2.5, x_pos - 0.5, x_pos - 0.5]
    y_box = [y_pos - 0.5, y_pos - 0.5, y_pos + 2.5, y_pos + 2.5, y_pos - 0.5]

    # Assign updates only to Traces 2, 3, and 4
    frame = go.Frame(
        name=str(step_idx),
        data=[
            go.Heatmap(z=curr_fm),      # Updates Feature Map colors
            go.Scatter(text=curr_text), # Updates Feature Map text
            go.Scatter(x=x_box, y=y_box)# Updates Bounding Box position
        ],
        traces=[2, 3, 4] 
    )
    frames.append(frame)

    # Link the frame to a step on the slider
    step_dict = dict(
        method="animate",
        args=[[str(step_idx)], {"frame": {"duration": 0, "redraw": True}, "mode": "immediate", "transition": {"duration": 0}}],
        label=f"Row {y_pos}, Col {x_pos}"
    )
    steps.append(step_dict)

fig.frames = frames

# --- 4. Layout and Styling ---
fig.update_layout(
    width=900, height=500, plot_bgcolor='white', showlegend=False,
    sliders=[dict(active=0, currentvalue={"prefix": "Current Step: "}, pad={"t": 40}, steps=steps)]
)

# Fix axes so they behave like images (inverted Y-axis, square pixels, hide gridlines)
fig.update_xaxes(showticklabels=False, showgrid=False, zeroline=False, scaleanchor="y", scaleratio=1)
fig.update_yaxes(showticklabels=False, showgrid=False, zeroline=False, autorange="reversed")

fig.show()

# ── Pooling visualization ─────────────────────────────────────────────
feature = np.array([
    [1, 3, 2, 4],
    [5, 6, 1, 2],
    [7, 2, 3, 1],
    [0, 8, 4, 5],
], dtype=np.float32)

# 2×2 Max Pooling
max_pool = np.array([
    [max(feature[0,0], feature[0,1], feature[1,0], feature[1,1]),
     max(feature[0,2], feature[0,3], feature[1,2], feature[1,3])],
    [max(feature[2,0], feature[2,1], feature[3,0], feature[3,1]),
     max(feature[2,2], feature[2,3], feature[3,2], feature[3,3])],
])

# 2×2 Average Pooling
avg_pool = np.array([
    [np.mean(feature[0:2, 0:2]), np.mean(feature[0:2, 2:4])],
    [np.mean(feature[2:4, 0:2]), np.mean(feature[2:4, 2:4])],
])

fig, axes = plt.subplots(1, 3, figsize=(14, 4))
for ax, data, title in zip(axes, [feature, max_pool, avg_pool],
                            ['Feature Map (4×4)', 'Max Pool 2×2 → (2×2)', 'Avg Pool 2×2 → (2×2)']):
    ax.imshow(data, cmap='YlOrRd', vmin=0, vmax=8)
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            ax.text(j, i, f'{data[i,j]:.1f}', ha='center', va='center', fontsize=16, fontweight='bold')
    ax.set_title(title, fontsize=13)
    ax.axis('off')
plt.suptitle('Pooling Operations Illustrated', fontsize=15, fontweight='bold')
plt.tight_layout()
plt.show()

print('Max pooling keeps the STRONGEST response in each 2×2 window.')
print('Average pooling smooths things out — useful in some architectures.')

# ── Synthetic stripe dataset ──────────────────────────────────────────

def make_stripe_data(n_samples=400, img_size=8):
    """Create a tiny dataset of vertical vs horizontal stripes."""
    images = []
    labels = []
    for i in range(n_samples):
        # Start with random noise
        img = np.random.randn(img_size, img_size).astype(np.float32) * 0.2
        if i % 2 == 0:
            # Class 0: vertical stripe (columns 3-4 are bright)
            img[:, 3:5] += 1.0
            labels.append(0)
        else:
            # Class 1: horizontal stripe (rows 3-4 are bright)
            img[3:5, :] += 1.0
            labels.append(1)
        images.append(img)
    
    X = torch.tensor(np.array(images)).unsqueeze(1)  # (N, 1, 8, 8)
    y = torch.tensor(labels)
    return X, y

X_toy, y_toy = make_stripe_data(400)

# Train/test split
X_train_toy, X_test_toy = X_toy[:320], X_toy[320:]
y_train_toy, y_test_toy = y_toy[:320], y_toy[320:]

train_toy_loader = DataLoader(TensorDataset(X_train_toy, y_train_toy), batch_size=32, shuffle=True)
test_toy_loader  = DataLoader(TensorDataset(X_test_toy, y_test_toy), batch_size=32)

# Visualize a few samples
fig, axes = plt.subplots(2, 5, figsize=(12, 5))
for i, ax in enumerate(axes[0]):
    ax.imshow(X_train_toy[i*2, 0], cmap='gray')
    ax.set_title(f'Class 0 (vertical)', fontsize=10)
    ax.axis('off')
for i, ax in enumerate(axes[1]):
    ax.imshow(X_train_toy[i*2+1, 0], cmap='gray')
    ax.set_title(f'Class 1 (horizontal)', fontsize=10)
    ax.axis('off')
plt.suptitle('Toy Dataset: Vertical vs Horizontal Stripes', fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()

print(f'Training samples: {len(X_train_toy)}, Test samples: {len(X_test_toy)}')