Bias-Variance Decomposition for Regression Problems

In this example, we will see how to calculate the bias-variance decomposition for regression problems.

import pandas as pd

from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split

from mvtk.bias_variance import bias_variance_compute, bias_variance_mse
from mvtk.bias_variance.estimators import EstimatorWrapper

WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1701450845.494601       1 tfrt_cpu_pjrt_client.cc:349] TfrtCpuClient created.

random_state=123

Load the example dataset

housing = fetch_california_housing()
X = pd.DataFrame(housing.data, columns=housing.feature_names)
y = housing.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=random_state)

Scikit-Learn Example

from sklearn.linear_model import LinearRegression

from mvtk.bias_variance.estimators import SciKitLearnEstimatorWrapper

model_scikit = LinearRegression()

Need to instantiate a wrapper class for usage by the bias variance calculation

model_scikit_wrapped = SciKitLearnEstimatorWrapper(model_scikit)

# Use wrapped estimator
avg_loss, avg_bias, avg_var, net_var = bias_variance_compute(model_scikit_wrapped, X_train, y_train, X_test, y_test, iterations=200,
                                                             random_state=random_state, decomp_fn=bias_variance_mse)

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

average loss: 0.59902430
average bias: 0.52119134
average variance: 0.07783295
net variance: 0.07783295

PyTorch Example

import torch
import torch.nn as nn

from mvtk.bias_variance.estimators import PyTorchEstimatorWrapper

X_train_torch = torch.FloatTensor(X_train.values)
X_test_torch = torch.FloatTensor(X_test.values)
y_train_torch = torch.FloatTensor(y_train).reshape(-1, 1)
y_test_torch = torch.FloatTensor(y_test).reshape(-1, 1)

class ModelPyTorch(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear1 = nn.Linear(8, 24)
        self.linear2 = nn.Linear(24, 12)
        self.linear3 = nn.Linear(12, 6)
        self.linear4 = nn.Linear(6, 1)

    def forward(self, x):
        x = self.linear1(x)
        x = self.linear2(x)
        x = self.linear3(x)
        x = self.linear4(x)
        return x

model_pytorch = ModelPyTorch()
optimizer = torch.optim.Adam(model_pytorch.parameters(), lr=0.001)
loss_fn = nn.MSELoss()

Need to instantiate a wrapper class for usage by the bias variance calculation

def optimizer_generator(x):
    return torch.optim.Adam(x.parameters(), lr=0.001)

model_pytorch_wrapped = PyTorchEstimatorWrapper(model_pytorch, optimizer_generator, loss_fn)

# Use wrapped estimator
avg_loss, avg_bias, avg_var, net_var = bias_variance_compute(model_pytorch_wrapped, X_train_torch, y_train_torch, X_test_torch, y_test,
                                                             iterations=200, random_state=random_state, decomp_fn=bias_variance_mse,
                                                             fit_kwargs={'epochs': 25})

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

average loss: 153.52321480
average bias: 5.05105724
average variance: 148.47215756
net variance: 148.47215756

TensorFlow example

import tensorflow as tf
from keras import initializers

from mvtk.bias_variance.estimators import TensorFlowEstimatorWrapper

model_tensorflow = tf.keras.Sequential([
    tf.keras.layers.Dense(64, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=0)),
    tf.keras.layers.Dense(64, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=0)),
    tf.keras.layers.Dense(1, kernel_initializer=initializers.glorot_uniform(seed=0))
])

model_tensorflow.compile(optimizer=tf.keras.optimizers.legacy.Adam(learning_rate=0.001),
             loss='mean_absolute_error',
             metrics=['mean_squared_error'])

model_tensorflow_wrapped = TensorFlowEstimatorWrapper(model_tensorflow)

# Use wrapped estimator
avg_loss, avg_bias, avg_var, net_var = bias_variance_compute(model_tensorflow_wrapped, X_train, y_train, X_test, y_test, iterations=200,
                                                             random_state=random_state, decomp_fn=bias_variance_mse,
                                                             fit_kwargs={'epochs': 25, 'batch_size': 5000, 'verbose': False},
                                                             predict_kwargs={'verbose': False})

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

average loss: 12.15213883
average bias: 3.08502204
average variance: 9.06711679
net variance: 9.06711679

We can run the same bias variance calculation in parallel for faster execution (in general for larger datasets and more intensive computations)

from mvtk.bias_variance import bias_variance_compute_parallel

avg_loss, avg_bias, avg_var, net_var = bias_variance_compute_parallel(model_tensorflow_wrapped, X_train, y_train, X_test, y_test,
                                                                      iterations=200, random_state=random_state,
                                                                      decomp_fn=bias_variance_mse,
                                                                      fit_kwargs={'epochs': 25, 'batch_size': 5000, 'verbose': False},
                                                                      predict_kwargs={'verbose': False})

print(f'average loss: {avg_loss:10.8f}')
print(f'average bias: {avg_bias:10.8f}')
print(f'average variance: {avg_var:10.8f}')
print(f'net variance: {net_var:10.8f}')

2023-12-01 12:15:52,912 INFO worker.py:1673 -- Started a local Ray instance.
(pid=73435) WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
(pid=73435) I0000 00:00:1701450956.584361       1 tfrt_cpu_pjrt_client.cc:349] TfrtCpuClient created.

average loss: 7.90420163
average bias: 4.06288774
average variance: 3.84131389
net variance: 3.84131389