import numpy as np
import pandas as pd
from sklearn.datasets import make_regression
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.preprocessing import StandardScaler
CS 307: Week 05
Regularization
# create some data
= make_regression(n_samples=100, n_features=20, noise=0.1, random_state=42) X, y
# view first two rows of the X data
0:2] X[
array([[-0.92216532, 1.87679581, 0.75698862, 0.27996863, 0.72576662,
0.48100923, 1.35563786, -1.2446547 , 0.4134349 , 0.86960592,
0.65436566, -1.12548905, 2.44575198, 0.12922118, 0.22388402,
1.49604431, -0.7737892 , -0.05558467, 0.10939479, -1.77872025],
[-0.08310557, -1.4575515 , -1.40631746, -0.1601328 , -0.79602586,
1.07600714, 0.76005596, -0.75215641, 0.08243975, -1.50472037,
-1.87517247, 0.67134008, 0.21319663, -0.75196933, 0.02131165,
1.34045045, -0.30920908, 0.11502608, -0.31905394, 0.31917451]])
# view some examples from the y data
0:5] y[
array([ 108.81742028, -250.567936 , 1.86765761, 127.84318259,
34.15127975])
# get a better look at the X data by temporarily displaying it as a pandas data frame
pd.DataFrame(X)
0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | -0.922165 | 1.876796 | 0.756989 | 0.279969 | 0.725767 | 0.481009 | 1.355638 | -1.244655 | 0.413435 | 0.869606 | 0.654366 | -1.125489 | 2.445752 | 0.129221 | 0.223884 | 1.496044 | -0.773789 | -0.055585 | 0.109395 | -1.778720 |
1 | -0.083106 | -1.457551 | -1.406317 | -0.160133 | -0.796026 | 1.076007 | 0.760056 | -0.752156 | 0.082440 | -1.504720 | -1.875172 | 0.671340 | 0.213197 | -0.751969 | 0.021312 | 1.340450 | -0.309209 | 0.115026 | -0.319054 | 0.319175 |
2 | 0.810808 | -1.662492 | -0.134309 | -0.308034 | -0.209222 | -1.683438 | -1.748532 | 1.126705 | 1.304340 | 0.793489 | -1.105705 | 0.779661 | 1.310309 | 1.395684 | -0.805870 | -0.410814 | 1.032546 | -0.214921 | -0.562168 | -1.090966 |
3 | 0.536653 | -0.756795 | -1.046911 | 0.455888 | 0.268592 | 1.528468 | 0.718953 | 1.501334 | 0.996048 | 1.185704 | 1.328194 | 2.165002 | -0.643518 | 0.927840 | 0.507836 | -0.250833 | -1.421811 | 0.556230 | 0.057013 | -0.322680 |
4 | 1.532739 | -0.401220 | 0.519347 | 1.451144 | 0.183342 | 2.189803 | 0.401712 | 0.012592 | 0.690144 | -0.108760 | 0.024510 | 0.959271 | 2.153182 | -0.767348 | -0.808298 | -0.773010 | 0.224092 | 0.497998 | 0.872321 | 0.097676 |
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
95 | -0.114736 | -0.334501 | -0.792521 | 2.122156 | -0.707669 | 0.443819 | 0.865755 | -0.653329 | -1.200296 | 0.504987 | -1.260884 | 1.032465 | -1.519370 | -0.484234 | 0.774634 | 0.404982 | -0.474945 | 0.917862 | 1.266911 | 1.765454 |
96 | -0.599375 | 0.622850 | -1.594428 | -1.534114 | 0.115675 | 1.179297 | 0.046981 | -0.142379 | -0.450065 | 0.005244 | 0.711615 | 1.277677 | 0.332314 | -0.748487 | 0.067518 | 0.514439 | -1.067620 | -1.124642 | 1.551152 | 0.120296 |
97 | -0.152470 | -1.331233 | 0.133541 | -0.006071 | -0.290275 | 0.267392 | 0.956702 | 0.507991 | -0.785989 | 0.708109 | 0.388579 | 0.838491 | 0.081829 | -0.098890 | 0.321698 | -2.152891 | -1.836205 | 2.493000 | 0.919076 | -1.103367 |
98 | -1.379618 | 0.513085 | -0.971657 | 1.188913 | -0.881875 | -0.163067 | 0.862393 | 0.516178 | 0.953125 | -0.626717 | 0.800410 | 0.708304 | 0.351448 | 1.070150 | -0.744903 | 0.431923 | 0.725096 | 0.754291 | -0.026521 | -0.641482 |
99 | -2.848543 | -1.119670 | 0.771699 | 0.076822 | -0.428115 | 1.500760 | -1.739714 | 1.160827 | -0.362441 | 1.148766 | -0.046921 | -1.282992 | 0.996267 | -0.493757 | 0.850222 | 0.346504 | -1.294681 | 0.477041 | -1.556582 | -0.467701 |
100 rows × 20 columns
# create different linear models (some with regularization!)
= LinearRegression()
linear = Lasso(alpha=0.1)
lasso = Ridge(alpha=0.1) ridge
Relevant documentation:
# fit the models
linear.fit(X, y)
lasso.fit(X, y) ridge.fit(X, y)
Ridge(alpha=0.1)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
Ridge(alpha=0.1)
linear.coef_
array([ 6.59316802e+00, 9.47572840e+01, 4.07080537e+01, -1.36954387e-03,
9.32945665e-03, -1.55281734e-02, 1.11188043e+01, 9.55121723e+01,
8.08254906e+01, 3.48859900e+01, 1.62428940e-02, 1.09659676e-02,
9.76125777e-03, 1.02690177e-02, -6.11540038e-03, 2.99366631e+01,
7.23584909e+00, 9.68705480e-03, -1.93391406e-03, 5.22797071e+01])
ridge.coef_
array([ 6.59850269e+00, 9.46618846e+01, 4.06454457e+01, 4.37714102e-03,
6.54059913e-03, -1.06479513e-02, 1.11069896e+01, 9.53732188e+01,
8.07112840e+01, 3.48957311e+01, 4.45154929e-04, 3.57365603e-02,
3.00167738e-02, 5.40170250e-03, -1.88067055e-02, 2.98957936e+01,
7.23859203e+00, -2.66565706e-02, -3.45743780e-02, 5.21972230e+01])
lasso.coef_
array([ 6.45386948, 94.67737779, 40.55794675, 0. , -0. ,
-0. , 10.98839003, 95.40225582, 80.67610577, 34.83198456,
-0. , 0. , 0. , -0. , -0. ,
29.82848069, 7.12581798, -0. , 0. , 52.17111821])
/ linear.coef_ lasso.coef_
array([ 0.97887229, 0.99915673, 0.9963126 , -0. , -0. ,
0. , 0.98827084, 0.99884919, 0.99815176, 0.99845194,
-0. , 0. , 0. , -0. , 0. ,
0.99638629, 0.98479362, -0. , -0. , 0.99792292])
= Lasso(alpha=0)
lasso_0 lasso_0.fit(X, y)
/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/base.py:1151: UserWarning: With alpha=0, this algorithm does not converge well. You are advised to use the LinearRegression estimator
return fit_method(estimator, *args, **kwargs)
/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_coordinate_descent.py:628: UserWarning: Coordinate descent with no regularization may lead to unexpected results and is discouraged.
model = cd_fast.enet_coordinate_descent(
Lasso(alpha=0)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
Lasso(alpha=0)
lasso_0.coef_
array([ 6.59220650e+00, 9.47578704e+01, 4.07086530e+01, 1.67942913e-03,
7.04365608e-03, -1.08366993e-02, 1.11190745e+01, 9.55172343e+01,
8.08290446e+01, 3.48823135e+01, 1.51450701e-02, 6.76300320e-03,
6.58339664e-03, 1.01400600e-02, -4.48418689e-03, 2.99366761e+01,
7.23587178e+00, 1.20404224e-02, 6.35862604e-04, 5.22790377e+01])
= Lasso(alpha=0.1)
lasso_01 = Lasso(alpha=5) lasso_05
lasso_01.fit(X, y) lasso_05.fit(X, y)
Lasso(alpha=5)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
Lasso(alpha=5)
lasso_01.coef_
array([ 6.45386948, 94.67737779, 40.55794675, 0. , -0. ,
-0. , 10.98839003, 95.40225582, 80.67610577, 34.83198456,
-0. , 0. , 0. , -0. , -0. ,
29.82848069, 7.12581798, -0. , 0. , 52.17111821])
lasso_05.coef_
array([ 0. , 90.82037065, 33.18439832, 0. , -0. ,
-0. , 4.90698647, 90.05810943, 73.22163014, 32.39510376,
-0. , 0. , 0. , -0. , 0. ,
24.45096119, 1.92062117, -0. , 0. , 46.96663538])
= StandardScaler()
scaler
scaler.fit(X)= scaler.transform(X) X_scaled
=0) np.mean(X, axis
array([ 0.06648265, -0.09066858, 0.14879335, -0.0092515 , 0.06688254,
0.13867757, -0.020263 , 0.1355665 , 0.04824566, 0.02152732,
-0.06736594, 0.11621231, 0.18151189, -0.02125737, 0.14082977,
0.16464492, -0.08393592, 0.01139174, -0.01378138, -0.0325596 ])
=0) np.mean(X_scaled, axis
array([-4.44089210e-17, -4.21884749e-17, -2.55351296e-17, -1.84574578e-17,
9.99200722e-18, 3.99680289e-17, -5.10702591e-17, -4.66293670e-17,
-8.38218384e-17, 2.22044605e-18, -3.92047506e-18, 4.44089210e-17,
3.66373598e-17, 3.21964677e-17, -3.44169138e-17, 2.35922393e-17,
-4.44089210e-18, 2.55351296e-17, 5.77315973e-17, 3.38618023e-17])
=0) np.std(X, axis
array([1.02482769, 0.97262509, 0.92892846, 0.89105338, 0.92304624,
0.91902508, 0.97366298, 0.94131669, 0.89996273, 1.07352597,
0.84522696, 1.01952759, 1.04023857, 1.04776952, 0.91703382,
1.08312224, 1.05773266, 1.1802187 , 1.00197408, 0.88057435])
=0) np.std(X_scaled, axis
array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
1., 1., 1.])
lasso.fit(X_scaled, y)
Lasso(alpha=0.1)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
Lasso(alpha=0.1)
# WRONG
lasso.predict(X) # correct lasso.predict(X_scaled)
array([ 108.56929274, -250.27007327, 1.74711702, 127.74731283,
34.16484902, 6.73089519, -47.99671007, -323.75282966,
40.71830457, 12.39299331, -233.2758167 , -68.89474373,
-95.14530018, 55.75266526, -25.81229695, 77.54586244,
88.98514129, -200.10765794, 49.19701855, 421.46438405,
157.23398526, 87.67924179, -36.15995542, 216.57697103,
340.00532386, -275.91290738, 257.12555842, -374.03964729,
357.20158352, -244.88762318, 93.67700784, -39.95057358,
61.98410136, 212.02109074, 265.26394524, 281.77244047,
318.53883286, 28.62306877, -22.37384227, 234.1955855 ,
-238.8564661 , 280.59818363, -49.02400195, -89.68866067,
-60.59043932, 34.04378327, 147.98817117, 243.56423325,
-14.29619749, 79.54075017, -45.42211395, -36.5241835 ,
29.71181164, -176.91121525, 96.89249288, 52.16684761,
114.5794129 , -333.86642314, 226.29054183, -106.73890637,
-6.6769494 , 162.63925058, -30.94403975, -21.76185466,
-19.87450714, -227.84122694, 87.90333805, 11.65702773,
209.70645989, 214.75616587, -108.581903 , 199.07494731,
-130.09302298, 33.6694303 , -58.72737906, -100.74761299,
-254.15146038, -9.07234502, 157.97000597, -444.49241247,
6.22181382, -1.07204728, 59.76748702, -58.99329766,
-71.9214518 , 207.05877426, -20.81839889, -265.38705602,
97.88748469, 114.24112507, 144.72937082, -55.47105068,
86.01322255, 300.68411823, 22.14183285, -95.77237868,
-44.71399397, -236.20063655, 98.68319317, -13.91650175])
Ensemble Methods
import numpy as np
from sklearn.datasets import make_regression
from sklearn.tree import DecisionTreeRegressor
import matplotlib.pyplot as plt
= make_regression(n_samples=100, n_features=5, noise=0.3, random_state=42) X, y
= DecisionTreeRegressor(max_depth=3)
dt dt.fit(X, y)
DecisionTreeRegressor(max_depth=3)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
DecisionTreeRegressor(max_depth=3)
= dt.predict(X) preds
- preds) ** 2)) np.sqrt(np.mean((y
72.97884422360983
# setup figure
= plt.subplots(1, 1)
fig, (ax) 5, 5)
fig.set_size_inches(100)
fig.set_dpi(
# add overall title
'Predicted versus Actual')
fig.suptitle(
# create subplot for linear model
="dodgerblue")
ax.scatter(y, preds, color"Actual")
ax.set_xlabel("Predicted")
ax.set_ylabel(True, linestyle='--', color='lightgrey')
ax.grid(0, 0), slope=1, color="black")
ax.axline((
# shot plot
plt.show()
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
= RandomForestRegressor(max_features=5)
bagged = RandomForestRegressor(max_features=2)
rf = GradientBoostingRegressor() boost
bagged.fit(X, y)
rf.fit(X, y) boost.fit(X, y)
GradientBoostingRegressor()In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
GradientBoostingRegressor()
= dt.predict(X)
preds_dt = bagged.predict(X)
preds_bagged = rf.predict(X)
preds_rf = boost.predict(X) preds_boost
# setup figure
= plt.subplots(1, 3)
fig, (ax1, ax2, ax3) 15, 5)
fig.set_size_inches(100)
fig.set_dpi(
# add overall title
'Predicted versus Actual')
fig.suptitle(
# create subplot for dt
"Single Decision Tree")
ax1.set_title(="dodgerblue")
ax1.scatter(y, preds_dt, color"Actual")
ax1.set_xlabel("Predicted")
ax1.set_ylabel(True, linestyle='--', color='lightgrey')
ax1.grid(0, 0), slope=1, color="black")
ax1.axline((
# create subplot for rf
"Random Forest")
ax2.set_title(="dodgerblue")
ax2.scatter(y, preds_rf, color"Actual")
ax2.set_xlabel("Predicted")
ax2.set_ylabel(True, linestyle='--', color='lightgrey')
ax2.grid(0, 0), slope=1, color="black")
ax2.axline((
# create subplot for boost
"Boosted Trees")
ax3.set_title(="dodgerblue")
ax3.scatter(y, preds_boost, color"Actual")
ax3.set_xlabel("Predicted")
ax3.set_ylabel(True, linestyle='--', color='lightgrey')
ax3.grid(0, 0), slope=1, color="black")
ax3.axline((
# shot plot
plt.show()