Bias-Variance Decomposition for Classification Problems

In this example, we will see how to calculate the bias-variance decomposition for classification problems.

import pandas as pd

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

from mvtk.bias_variance import bias_variance_compute, bias_variance_0_1_loss
from mvtk.bias_variance.estimators import EstimatorWrapper

WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1701450732.304993       1 tfrt_cpu_pjrt_client.cc:349] TfrtCpuClient created.

random_state=123

Load the example dataset

iris = load_iris()
X = pd.DataFrame(iris.data, columns=iris.feature_names)
y = iris.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=random_state)

Scikit-Learn Example

from sklearn.tree import DecisionTreeClassifier

from mvtk.bias_variance.estimators import SciKitLearnEstimatorWrapper

model_scikit = DecisionTreeClassifier(random_state=random_state)

Need to instantiate a wrapper class for usage by the bias variance calculation

model_scikit_wrapped = SciKitLearnEstimatorWrapper(model_scikit)

# Use wrapped estimator
avg_loss, avg_bias, avg_var, net_var = bias_variance_compute(model_scikit_wrapped, X_train, y_train, X_test, y_test, iterations=200,
                                                             random_state=random_state, decomp_fn=bias_variance_0_1_loss)

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

average loss: 0.06066667
average bias: 0.04444444
average variance: 0.03311111
net variance: 0.01622222

PyTorch Example

import torch
import torch.nn as nn

from mvtk.bias_variance.estimators import PyTorchEstimatorWrapper

X_train_torch = torch.FloatTensor(X_train.values)
X_test_torch = torch.FloatTensor(X_test.values)
y_train_torch = torch.LongTensor(y_train)
y_test_torch = torch.LongTensor(y_test)

class ModelPyTorch(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear1 = nn.Linear(4, 25)
        self.linear2 = nn.Linear(25, 3)

    def forward(self, x):
        x = self.linear1(x)
        x = self.linear2(x)
        return x

model_pytorch = ModelPyTorch()
optimizer = torch.optim.Adam(model_pytorch.parameters(), lr=0.01)
loss_fn = nn.CrossEntropyLoss()

Need to instantiate a wrapper class for usage by the bias variance calculation

def optimizer_generator(x):
    return torch.optim.Adam(x.parameters(), lr=0.01)

model_pytorch_wrapped = PyTorchEstimatorWrapper(model_pytorch, optimizer_generator, loss_fn)

# Use wrapped estimator
avg_loss, avg_bias, avg_var, net_var = bias_variance_compute(model_pytorch_wrapped, X_train_torch, y_train_torch, X_test_torch, y_test,
                                                             iterations=200, random_state=random_state, decomp_fn=bias_variance_0_1_loss,
                                                             fit_kwargs={'epochs': 25})

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

average loss: 0.19355556
average bias: 0.02222222
average variance: 0.19177778
net variance: 0.17133333

Same idea with TensorFlow models

import tensorflow as tf
from keras import initializers

from mvtk.bias_variance.estimators import TensorFlowEstimatorWrapper

model_tensorflow = tf.keras.Sequential([
    tf.keras.layers.Dense(10, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=0)),
    tf.keras.layers.Dense(10, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=0)),
    tf.keras.layers.Dense(3, activation='softmax', kernel_initializer=initializers.glorot_uniform(seed=0))
])

model_tensorflow.compile(optimizer='rmsprop',
             loss='sparse_categorical_crossentropy',
             metrics=['accuracy'])

Need to instantiate a wrapper class for usage by the bias variance calculation

model_tensorflow_wrapped = TensorFlowEstimatorWrapper(model_tensorflow)

# Use wrapped estimator
avg_loss, avg_bias, avg_var, net_var = bias_variance_compute(model_tensorflow_wrapped, X_train, y_train, X_test, y_test, iterations=200,
                                                             random_state=random_state, decomp_fn=bias_variance_0_1_loss,
                                                             fit_kwargs={'epochs': 25, 'batch_size': 50, 'verbose': False},
                                                             predict_kwargs={'verbose': False})

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

average loss: 0.18800000
average bias: 0.17777778
average variance: 0.08988889
net variance: 0.01022222

We can run the same bias variance calculation in parallel for faster execution (in general for larger datasets and more intensive computations)

from mvtk.bias_variance import bias_variance_compute_parallel

avg_loss, avg_bias, avg_var, net_var = bias_variance_compute_parallel(model_tensorflow_wrapped, X_train, y_train, X_test, y_test,
                                                                      iterations=200, random_state=random_state,
                                                                      decomp_fn=bias_variance_0_1_loss,
                                                                      fit_kwargs={'epochs': 25, 'batch_size': 50, 'verbose': False},
                                                                      predict_kwargs={'verbose': False})

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

average loss: 0.18133333
average bias: 0.11111111
average variance: 0.13000000
net variance: 0.07022222