Neural Network

Notebook producing the figures in Linear Regression to Neural Networks. Chapter on Neural Networks. We are going to fit a Neural Network model to try and classify some Penguin species data. We are going to produce this plot:

Let's go!

For all source files, see Github repository.

import shutil
import tempfile
from time import time
import warnings
from warnings import filterwarnings

import IPython
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from IPython import get_ipython
from IPython.display import HTML, set_matplotlib_formats
from matplotlib.animation import FuncAnimation
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.exceptions import ConvergenceWarning
from tqdm.notebook import tqdm

get_ipython().run_line_magic('matplotlib', 'inline')
set_matplotlib_formats('svg')
get_ipython().system('python --version')

Python 3.9.2

with warnings.catch_warnings():
    warnings.filterwarnings("ignore", category=ConvergenceWarning)
filterwarnings('ignore')

Load Penguin dataset.

data = sns.load_dataset('penguins')
data = data.dropna()
data.dtypes

species               object
island                object
bill_length_mm       float64
bill_depth_mm        float64
flipper_length_mm    float64
body_mass_g          float64
sex                   object
dtype: object

Make column for only indicating Chinstrap yes/no.

peng = lambda x: 'Chinstrap' if x == 'Chinstrap' else 'Other'
data['Penguin'] = data['species'].apply(peng)

Training/testing cross validation split. Reserve 1/4th of data for testing.

rs = np.random.RandomState(34)
test = rs.choice(data.index, len(data) // 4)
train = data.index[~data.index.isin(test)]
data.loc[train, 'subset'] = 'Train'
data.loc[test, 'subset'] = 'Test'
data_train = data[data['subset'] == 'Train']
data_test = data[data['subset'] == 'Test']
data.groupby('subset').count()[['species']]

	species
subset
Test	78
Train	255

blue_colors = sns.color_palette("Paired", n_colors=2)
sns.scatterplot(data=data_train, x='bill_depth_mm', y='bill_length_mm',
                hue='Penguin', palette=blue_colors, style='subset',
                style_order=['Train', 'Test'])

<AxesSubplot:xlabel='bill_depth_mm', ylabel='bill_length_mm'>

X       = data[['bill_depth_mm', 'bill_length_mm']].values
X_train = data_train[['bill_depth_mm', 'bill_length_mm']].values
y_train = data_train['Penguin'].values
X_test  = data_test[['bill_depth_mm', 'bill_length_mm']].values
y_test  = data_test['Penguin'].values

Let's try to classify Chinstraps using a Neural Network. We'll use sklearn for this. Try a fit:

clf = MLPClassifier()
clf.fit(X_train, y_train)
clf.n_iter_, clf.classes_, clf.loss_

(200, array(['Chinstrap', 'Other'], dtype='<U9'), 0.40034055245196065)

relu = lambda z: max(0, z)

plt.figure(figsize=(5, 4))
plt.plot(range(-9, 9), [relu(z) for z in range(-9, 9)])
plt.xticks([-5, 0, 5])
plt.yticks([0, 2, 4, 6, 8])
plt.grid()
plt.xlabel('z')
plt.ylabel('$\sigma(z)$')
plt.title('ReLU activation function')
plt.savefig('images/relu.svg')

Test classifier score() function.

clf.fit(X_train, y_train)
clf.score(X_test, y_test)

0.8076923076923077

Next, we create a function to predict our classifier over a grid points, to be used for plotting our decision regions later.

def apply_over_grid(X, clf, *args):
    xaxis = np.linspace(X[:, 0].min(), X[:, 0].max(), 100)
    yaxis = np.linspace(X[:, 1].min(), X[:, 1].max(), 100)
    xx, yy = np.meshgrid(xaxis, yaxis)
    zz = np.dstack([xx, yy])
    ax1, ax2, ax3 = zz.shape
    X = zz.reshape((ax1 * ax2, ax3))
    zz = clf(X, *args)
    zz = zz.reshape((ax1, ax2))
    return xx, yy, zz
xx, yy, zz = apply_over_grid(X, lambda X: X.sum(axis=1))
X.shape, xx.shape, yy.shape, zz.shape

((333, 2), (100, 100), (100, 100), (100, 100))

Wrap our clf object in a predictor variable.

predictor = lambda clf: lambda X: clf.predict_proba(X)[:, 0]
predictor(clf)([[2, 3]])

array([0.13565336])

Run some couple thousand iterations using our Neural Network. It has a rather 'arbitrary' architecture of 3 layers of 5 nodes each - should be enough to capture the complexity of this dataset.

max_iter = 16
iterations = 2400
n_fits = iterations // max_iter
clf = MLPClassifier(
    hidden_layer_sizes=(5, 5, 5),
    alpha=0.0005,
    learning_rate_init=0.001,
    max_iter=max_iter,
    n_iter_no_change=max_iter,
    random_state=33,
    tol=1e-15,
    warm_start=True,
    solver='sgd',
    learning_rate='constant'
)

records = []
zzz = []
pbar = tqdm(range(n_fits))
iters = 0
for i in pbar:
    clf.fit(X_train, y_train)
    record = { 'Loss': clf.loss_, 'Iteration': clf.n_iter_, 'i': i }
    acc = clf.score(X_train, y_train)
    records.append({ **record, 'Acc': acc, 'Subset': 'Train' })
    acc = clf.score(X_test, y_test)
    records.append({ **record, 'Acc': acc, 'Subset': 'Test' })
    
    _, _, zz = apply_over_grid(X, predictor(clf))
    zzz.append(zz)
    pbar.set_description(f'n_iter={clf.n_iter_}')

Collect results in a DataFrame and plot loss and accuracy.

results = pd.DataFrame.from_records(records, index='i')
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 3.5))
sns.lineplot(data=results, x='Iteration', y='Acc', hue='Subset', ax=ax1)
sns.lineplot(data=results, x='Iteration', y='Loss', ax=ax2)
plt.suptitle('Neural Network penguin classification accuracy/loss')
plt.savefig('images/neural-metrics.svg')

Plot decision region of last fit.

def decision_region(ax, zz):
    plt.contourf(xx, yy, zz, alpha=0.8,
                           levels=[0.495, 0.505], colors=['red'])
    plt.contourf(xx, yy, zz, alpha=0.4, cmap='Blues')
    sns.scatterplot(data=data_train, x='bill_depth_mm', y='bill_length_mm',
                    hue='Penguin', palette=blue_colors, ax=ax)
    sns.scatterplot(data=data_test, x='bill_depth_mm', y='bill_length_mm',
                    hue='Penguin', palette=blue_colors, ax=ax,
                    style='subset', style_order=['Train', 'Test'],
                    legend=False)
    
fig = plt.figure()
ax = fig.gca()
decision_region(ax, zzz[-1])
plt.title(f'train (⚫︎)=232, test (×)=2324')

Text(0.5, 1.0, 'train (⚫︎)=232, test (×)=2324')

We are going to create a range of images. Create a temporary folder for them.

folder = tempfile.mkdtemp()
print(f'Saving images to {folder}')

Saving images to /var/folders/34/923qqr696yz_ybx0dl171fn80000gn/T/tmpn6q212lq

frames = 150
assert(frames <= n_fits)
interval = n_fits // frames
idxs = np.arange(frames - 1) * interval
idxs = np.append(n_fits - 1, idxs)
f'{n_fits} fits, {frames} frames; interval of {interval}. Start at {idxs[0]}.'

'150 fits, 150 frames; interval of 1. Start at 149.'

Compute and save separate GIF images.

pbar = tqdm(total=frames)
def create_frame(frame, ax, zzz):
    ax.cla()

    i = idxs[frame]

    # decision region
    decision_region(ax, zzz[i])

    # plot title
    result = results[results.index == i]
    trn = result[result['Subset'] == 'Train']
    tst = result[result['Subset'] == 'Test']
    loss = result['Loss'].unique().item()
    acc, acc_test = trn['Acc'].item(), tst['Acc'].item()
    iteration = result['Iteration'].unique().item()
    plt.title(f'Neural Network fit iteration {iteration:04}/{iterations}\n'+
              f'Acc: train (⚫︎)={acc:.3f}, test (×)={acc_test:.3f}, '+
              f'loss: train={loss:.3f}')
    plt.savefig(f'{folder}/frame_{frame:03}.png')

    # progress bar
    pbar.update()
    
fig = plt.figure()
ax = fig.gca()
animation = FuncAnimation(fig, create_frame,
    frames=frames, fargs=(ax, zzz), interval=100) # => 10 fps
animated = animation.to_jshtml()
pbar.close()

Convert separate images into a GIF.

name = './images/neural-fit.gif'
get_ipython().system('convert -background white -alpha remove '+
    '-dispose Previous +antialias -layers OptimizePlus '+
    f'{folder}/*.png {name}')

Clean the temporary folder 💎

shutil.rmtree(folder)

Author

Code written by Jeroen Overschie. MIT licensed.