Classical Machine Learning

Linear & Logistic Regression

Linear Regression

Linear regression finds the best-fit line through data by minimizing the sum of squared errors. The model assumes a linear relationship: ŷ = w₁x₁ + w₂x₂ + ... + wₙxₙ + b.

Python
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.datasets import make_regression

# Generate synthetic data
X, y = make_regression(n_samples=500, n_features=3, noise=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Scikit-learn
model = LinearRegression()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)

print(f"Coefficients: {model.coef_.round(2)}")
print(f"Intercept:    {model.intercept_:.2f}")
print(f"R² Score:     {r2_score(y_test, y_pred):.4f}")
print(f"RMSE:         {np.sqrt(mean_squared_error(y_test, y_pred)):.2f}")

From Scratch: Gradient Descent

Python
class LinearRegressionGD:
    def __init__(self, lr=0.01, epochs=1000):
        self.lr = lr
        self.epochs = epochs

    def fit(self, X, y):
        m, n = X.shape
        self.w = np.zeros(n)
        self.b = 0
        self.losses = []

        for _ in range(self.epochs):
            y_pred = X @ self.w + self.b
            error = y_pred - y
            self.w -= self.lr * (1/m) * (X.T @ error)
            self.b -= self.lr * (1/m) * np.sum(error)
            self.losses.append(np.mean(error**2))

    def predict(self, X):
        return X @ self.w + self.b

Logistic Regression

Logistic regression applies the sigmoid function σ(z) = 1/(1+e⁻ᶻ) to convert linear output into a probability between 0 and 1. It uses binary cross-entropy loss instead of MSE.

Python
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer
from sklearn.metrics import accuracy_score, classification_report

data = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target, test_size=0.2, random_state=42)

clf = LogisticRegression(max_iter=5000)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)

print(f"Accuracy: {accuracy_score(y_test, y_pred):.2%}")
print(classification_report(y_test, y_pred, target_names=data.target_names))

Python
from sklearn.datasets import fetch_california_housing
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Ridge, Lasso

housing = fetch_california_housing()
X_train, X_test, y_train, y_test = train_test_split(
    housing.data, housing.target, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# Compare Linear, Ridge (L2), Lasso (L1)
for name, model in [("Linear", LinearRegression()),
                    ("Ridge", Ridge(alpha=1.0)),
                    ("Lasso", Lasso(alpha=0.1))]:
    model.fit(X_train_s, y_train)
    score = model.score(X_test_s, y_test)
    print(f"{name:8s} R²: {score:.4f}")

def sigmoid(z): return 1 / (1 + np.exp(-z))
def bce_loss(y, y_hat):
    eps = 1e-15
    y_hat = np.clip(y_hat, eps, 1-eps)
    return -np.mean(y*np.log(y_hat) + (1-y)*np.log(1-y_hat))

Decision Trees & Random Forests

Decision Trees

A decision tree recursively splits data by asking yes/no questions on features, choosing splits that maximize information gain (reduce impurity). It's one of the most interpretable ML models.

Python
from sklearn.tree import DecisionTreeClassifier, export_text
from sklearn.datasets import load_iris

iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.2, random_state=42)

tree = DecisionTreeClassifier(max_depth=3, random_state=42)
tree.fit(X_train, y_train)

print(f"Accuracy: {tree.score(X_test, y_test):.2%}")
print("\nTree Rules:")
print(export_text(tree, feature_names=iris.feature_names))

Random Forests

A random forest builds many decision trees on random subsets of data (bagging) and random subsets of features, then votes/averages their predictions. This dramatically reduces overfitting.

Python
from sklearn.ensemble import RandomForestClassifier
import pandas as pd

rf = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=42)
rf.fit(X_train, y_train)
print(f"RF Accuracy: {rf.score(X_test, y_test):.2%}")

# Feature importance — which features matter most?
importance = pd.Series(rf.feature_importances_, index=iris.feature_names)
print("\nFeature Importance:")
print(importance.sort_values(ascending=False))

Python
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
import numpy as np

# Simulate churn data
np.random.seed(42)
n = 1000
X = np.column_stack([
    np.random.randint(1, 72, n),         # tenure (months)
    np.random.uniform(20, 120, n),       # monthly charge
    np.random.randint(0, 10, n),          # support calls
    np.random.choice([0,1], n)            # has contract
])
# Churn more likely with short tenure, high charge, many calls
churn_prob = 1/(1 + np.exp(-(-2 + 0.05*(60-X[:,0]) + 0.02*X[:,1] + 0.3*X[:,2] - 1.5*X[:,3])))
y = (np.random.random(n) < churn_prob).astype(int)

X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2)
rf = RandomForestClassifier(n_estimators=200, max_depth=6)
rf.fit(X_tr, y_tr)
print(classification_report(y_te, rf.predict(X_te), target_names=['Stay','Churn']))

Support Vector Machines (SVMs)

Maximum Margin Classifier

SVM finds the hyperplane that maximizes the margin — the distance between the nearest data points of each class (support vectors) and the decision boundary. Larger margin = better generalization.

Python
from sklearn.svm import SVC
from sklearn.datasets import make_classification
from sklearn.preprocessing import StandardScaler

X, y = make_classification(n_samples=500, n_features=2,
    n_redundant=0, n_clusters_per_class=1, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Always scale for SVM!
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# Compare kernels
for kernel in ['linear', 'rbf', 'poly']:
    svm = SVC(kernel=kernel, C=1.0)
    svm.fit(X_train_s, y_train)
    acc = svm.score(X_test_s, y_test)
    print(f"{kernel:8s} → Accuracy: {acc:.2%}, Support vectors: {svm.n_support_}")

Python
from sklearn.datasets import load_digits

digits = load_digits()
X_tr, X_te, y_tr, y_te = train_test_split(digits.data, digits.target, test_size=0.2)
X_tr_s = StandardScaler().fit_transform(X_tr)
X_te_s = StandardScaler().fit_transform(X_te)

svm = SVC(kernel='rbf', C=10, gamma='scale')
svm.fit(X_tr_s, y_tr)
print(f"Digit classification accuracy: {svm.score(X_te_s, y_te):.2%}")

k-Nearest Neighbors (kNN)

Python
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_wine

wine = load_wine()
X_tr, X_te, y_tr, y_te = train_test_split(wine.data, wine.target, test_size=0.2)
X_tr_s = StandardScaler().fit_transform(X_tr)
X_te_s = StandardScaler().fit_transform(X_te)

# Find best k
for k in [1, 3, 5, 7, 11, 15]:
    knn = KNeighborsClassifier(n_neighbors=k)
    knn.fit(X_tr_s, y_tr)
    print(f"k={k:2d} → Accuracy: {knn.score(X_te_s, y_te):.2%}")
# k=1 overfits (memorizes), large k underfits (too smooth)

knn = KNeighborsClassifier(n_neighbors=5, weights='distance')
# Closer neighbors get weight = 1/distance
# This helps when the decision boundary is complex

Ensemble Methods: Boosting & Bagging

Bagging vs Boosting

Python
from sklearn.ensemble import GradientBoostingClassifier, AdaBoostClassifier
from sklearn.datasets import load_breast_cancer

data = load_breast_cancer()
X_tr, X_te, y_tr, y_te = train_test_split(data.data, data.target, test_size=0.2)

# Gradient Boosting
gb = GradientBoostingClassifier(n_estimators=100, learning_rate=0.1, max_depth=3)
gb.fit(X_tr, y_tr)
print(f"Gradient Boosting: {gb.score(X_te, y_te):.2%}")

# AdaBoost
ada = AdaBoostClassifier(n_estimators=100, learning_rate=0.5)
ada.fit(X_tr, y_tr)
print(f"AdaBoost:          {ada.score(X_te, y_te):.2%}")

XGBoost — Industry Standard

Python
# pip install xgboost
import xgboost as xgb

model = xgb.XGBClassifier(
    n_estimators=200, max_depth=4, learning_rate=0.1,
    subsample=0.8, colsample_bytree=0.8,
    eval_metric='logloss', random_state=42
)
model.fit(X_tr, y_tr, eval_set=[(X_te, y_te)], verbose=0)
print(f"XGBoost: {model.score(X_te, y_te):.2%}")

# Feature importance
xgb.plot_importance(model, max_num_features=10)

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
models = {
    "RF": RandomForestClassifier(n_estimators=100),
    "GBM": GradientBoostingClassifier(n_estimators=100),
    "XGB": xgb.XGBClassifier(n_estimators=100, eval_metric='logloss')
}
for name, m in models.items():
    m.fit(X_tr, y_tr)
    print(f"{name}: {m.score(X_te, y_te):.2%}")

Naive Bayes Classifiers

Python
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer

# Text classification — Spam detection
emails = [
    "Win a free iPhone now click here",
    "Meeting scheduled for tomorrow 3pm",
    "Congratulations you won $1000 prize",
    "Please review the attached report",
    "Free trial offer limited time only",
    "Lunch meeting moved to 2pm",
    "Claim your reward now act fast",
    "Updated project timeline attached",
]
labels = [1,0,1,0,1,0,1,0]  # 1=spam, 0=ham

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(emails)

clf = MultinomialNB()
clf.fit(X, labels)

test = ["Free prize winner click now", "Project deadline is Friday"]
X_test = vectorizer.transform(test)
print(clf.predict(X_test))  # [1, 0] — spam, ham ✓

Clustering: k-Means & DBSCAN

k-Means Clustering

k-Means partitions data into k clusters by iteratively: (1) assigning each point to the nearest centroid, (2) moving centroids to the mean of assigned points. Repeats until convergence.

Python
from sklearn.cluster import KMeans, DBSCAN
from sklearn.metrics import silhouette_score
from sklearn.datasets import make_blobs

X, y_true = make_blobs(n_samples=500, centers=4, cluster_std=0.8, random_state=42)

# Elbow Method — find optimal k
inertias = []
for k in range(2, 10):
    km = KMeans(n_clusters=k, random_state=42, n_init=10)
    km.fit(X)
    inertias.append(km.inertia_)
    sil = silhouette_score(X, km.labels_)
    print(f"k={k}: inertia={km.inertia_:.0f}, silhouette={sil:.3f}")

# Best model
km = KMeans(n_clusters=4, random_state=42, n_init=10)
km.fit(X)
print(f"\nCluster sizes: {np.bincount(km.labels_)}")

DBSCAN — Density-Based Clustering

Unlike k-Means, DBSCAN doesn't require specifying k. It finds clusters of arbitrary shape by identifying dense regions separated by sparse areas. Points in sparse regions are labeled as noise.

Python
from sklearn.cluster import DBSCAN
from sklearn.datasets import make_moons

X_moons, _ = make_moons(n_samples=300, noise=0.1)

# k-Means fails on non-spherical data
km = KMeans(n_clusters=2, n_init=10).fit(X_moons)
print(f"k-Means silhouette: {silhouette_score(X_moons, km.labels_):.3f}")

# DBSCAN handles it perfectly
db = DBSCAN(eps=0.2, min_samples=5).fit(X_moons)
print(f"DBSCAN silhouette:  {silhouette_score(X_moons, db.labels_):.3f}")
print(f"Noise points: {(db.labels_ == -1).sum()}")

Python
# Simulate customer data
np.random.seed(42)
customers = np.column_stack([
    np.concatenate([np.random.normal(30,10,100), np.random.normal(60,8,100), np.random.normal(45,12,100)]),
    np.concatenate([np.random.normal(2000,500,100), np.random.normal(8000,1500,100), np.random.normal(4500,1000,100)])
])

scaler = StandardScaler()
X_scaled = scaler.fit_transform(customers)

km = KMeans(n_clusters=3, random_state=42, n_init=10)
labels = km.fit_predict(X_scaled)

for i in range(3):
    mask = labels == i
    avg_age = customers[mask, 0].mean()
    avg_spend = customers[mask, 1].mean()
    print(f"Segment {i}: {mask.sum()} customers, "
          f"avg age={avg_age:.0f}, avg spend=${avg_spend:,.0f}")

def kmeans(X, k, max_iter=100):
    centroids = X[np.random.choice(len(X), k, replace=False)]
    for _ in range(max_iter):
        dists = np.linalg.norm(X[:, None] - centroids, axis=2)
        labels = np.argmin(dists, axis=1)
        new_centroids = np.array([X[labels==i].mean(axis=0) for i in range(k)])
        if np.allclose(centroids, new_centroids): break
        centroids = new_centroids
    return labels, centroids

Dimensionality Reduction: PCA, t-SNE, UMAP

PCA — Principal Component Analysis

PCA finds the directions of maximum variance in your data and projects onto those axes. It reduces dimensions while retaining the most information possible.

Python
from sklearn.decomposition import PCA
from sklearn.datasets import load_digits

digits = load_digits()  # 64 features (8×8 pixels)

pca = PCA(n_components=2)
X_2d = pca.fit_transform(digits.data)
print(f"Original: {digits.data.shape}")       # (1797, 64)
print(f"Reduced:  {X_2d.shape}")                # (1797, 2)
print(f"Variance retained: {pca.explained_variance_ratio_.sum():.1%}")

# How many components to keep 95% variance?
pca_95 = PCA(n_components=0.95)
X_reduced = pca_95.fit_transform(digits.data)
print(f"Components for 95%: {pca_95.n_components_}")  # ~28 of 64

t-SNE & UMAP for Visualization

Python
from sklearn.manifold import TSNE

# t-SNE — best for 2D visualization of clusters
tsne = TSNE(n_components=2, perplexity=30, random_state=42)
X_tsne = tsne.fit_transform(digits.data)

# UMAP — faster, preserves global structure better
# pip install umap-learn
# import umap
# reducer = umap.UMAP(n_components=2)
# X_umap = reducer.fit_transform(digits.data)

Python
from sklearn.manifold import TSNE
from sklearn.datasets import load_digits
import matplotlib.pyplot as plt

digits = load_digits()
tsne = TSNE(n_components=2, perplexity=30, random_state=42)
X_vis = tsne.fit_transform(digits.data)

plt.figure(figsize=(10,8))
scatter = plt.scatter(X_vis[:,0], X_vis[:,1], c=digits.target,
                      cmap='tab10', alpha=0.7, s=10)
plt.colorbar(scatter, label='Digit')
plt.title('MNIST Digits — t-SNE Visualization')
plt.savefig('mnist_tsne.png', dpi=150)
# Each digit forms a distinct cluster!

Gaussian Mixture Models & Autoencoders

Gaussian Mixture Models

GMM assumes data comes from a mixture of k Gaussian distributions. Unlike k-Means (hard assignment), GMM gives probabilistic membership — each point has a probability of belonging to each cluster.

Python
from sklearn.mixture import GaussianMixture
from sklearn.datasets import make_blobs

X, _ = make_blobs(n_samples=500, centers=3, cluster_std=1.5, random_state=42)

gmm = GaussianMixture(n_components=3, random_state=42)
gmm.fit(X)
labels = gmm.predict(X)
probs = gmm.predict_proba(X)  # Soft assignment!

print(f"Point 0 probabilities: {probs[0].round(3)}")
# e.g., [0.001, 0.002, 0.997] — 99.7% cluster 2

# Model selection with BIC
for k in range(2, 7):
    g = GaussianMixture(n_components=k, random_state=42).fit(X)
    print(f"k={k}: BIC={g.bic(X):.0f}")  # Lower is better

Autoencoders (Concept)

An autoencoder is a neural network that learns to compress data into a low-dimensional "bottleneck" representation and reconstruct it. The bottleneck captures the most important features — similar to PCA but non-linear.

Autoencoders are used for: dimensionality reduction, denoising, anomaly detection (high reconstruction error = anomaly), and generative models (VAEs).

Anomaly Detection

Python
from sklearn.ensemble import IsolationForest
from sklearn.svm import OneClassSVM
import numpy as np

# Normal data + injected anomalies
np.random.seed(42)
X_normal = np.random.randn(500, 2)
X_anomaly = np.random.uniform(-6, 6, (20, 2))
X = np.vstack([X_normal, X_anomaly])

# Isolation Forest — isolates anomalies via random splits
iso = IsolationForest(contamination=0.05, random_state=42)
preds = iso.fit_predict(X)  # 1=normal, -1=anomaly
print(f"Isolation Forest: {(preds==-1).sum()} anomalies detected")

# One-Class SVM — learns boundary of normal data
ocsvm = OneClassSVM(kernel='rbf', nu=0.05)
preds_svm = ocsvm.fit_predict(X)
print(f"One-Class SVM: {(preds_svm==-1).sum()} anomalies detected")

# Statistical: Z-score
z_scores = np.abs((X - X.mean(axis=0)) / X.std(axis=0))
stat_anomalies = (z_scores > 3).any(axis=1)
print(f"Z-score (>3σ): {stat_anomalies.sum()} anomalies detected")

Python
from sklearn.ensemble import IsolationForest
from sklearn.metrics import classification_report

# Simulate transaction data (normally distributed amounts)
np.random.seed(42)
normal_txns = np.column_stack([
    np.random.normal(50, 20, 9900),    # amount
    np.random.normal(5, 2, 9900),      # frequency
    np.random.normal(100, 30, 9900),   # distance_from_home
])
fraud_txns = np.column_stack([
    np.random.normal(500, 200, 100),   # large amounts
    np.random.normal(20, 5, 100),      # high frequency
    np.random.normal(500, 150, 100),   # far from home
])
X = np.vstack([normal_txns, fraud_txns])
y_true = np.array([1]*9900 + [-1]*100)

iso = IsolationForest(contamination=0.02, random_state=42)
y_pred = iso.fit_predict(X)

print(classification_report(y_true, y_pred, target_names=['Fraud', 'Normal']))

Train/Validation/Test Splits & Cross-Validation

Data Splitting Strategies

Python
from sklearn.model_selection import (cross_val_score, StratifiedKFold,
                                     TimeSeriesSplit)
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer

data = load_breast_cancer()
X, y = data.data, data.target

# Stratified k-Fold (preserves class distribution)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
rf = RandomForestClassifier(n_estimators=100, random_state=42)

scores = cross_val_score(rf, X, y, cv=skf, scoring='accuracy')
print(f"5-Fold CV: {scores.mean():.4f} ± {scores.std():.4f}")
print(f"Per fold: {scores.round(4)}")

# Time Series Split — never look into the future!
tscv = TimeSeriesSplit(n_splits=5)
for i, (train_idx, test_idx) in enumerate(tscv.split(X)):
    print(f"Fold {i+1}: Train={len(train_idx)}, Test={len(test_idx)}")

Metrics, Bias-Variance & Regularization

Classification Metrics

Python
from sklearn.metrics import (confusion_matrix, classification_report,
                             roc_auc_score, roc_curve)
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer

data = load_breast_cancer()
X_tr, X_te, y_tr, y_te = train_test_split(data.data, data.target, test_size=0.2)

clf = LogisticRegression(max_iter=5000)
clf.fit(X_tr, y_tr)
y_pred = clf.predict(X_te)
y_proba = clf.predict_proba(X_te)[:, 1]

print("Confusion Matrix:")
print(confusion_matrix(y_te, y_pred))
print(f"\nAUC-ROC: {roc_auc_score(y_te, y_proba):.4f}")
print(classification_report(y_te, y_pred))

Bias-Variance Tradeoff

High bias (underfitting): Model is too simple — consistently wrong. Fix: use more complex model, add features. High variance (overfitting): Model memorizes training data — great on train, poor on test. Fix: more data, regularization, simpler model.

Regularization

Python
from sklearn.linear_model import Ridge, Lasso
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.pipeline import make_pipeline

# Demonstrating overfitting and regularization
np.random.seed(42)
X = np.sort(np.random.uniform(0, 1, 30)).reshape(-1, 1)
y = np.sin(2*np.pi*X).ravel() + np.random.normal(0, 0.2, 30)

# Underfit (degree 1), Good fit (degree 3), Overfit (degree 15)
for degree in [1, 3, 15]:
    model = make_pipeline(
        PolynomialFeatures(degree),
        Ridge(alpha=0.01 if degree < 15 else 1.0)
    )
    model.fit(X, y)
    print(f"Degree {degree:2d}: Train R²={model.score(X,y):.4f}")

Python
from sklearn.model_selection import learning_curve

def plot_learning_curve(model, X, y):
    train_sizes, train_scores, val_scores = learning_curve(
        model, X, y, cv=5, train_sizes=np.linspace(0.1, 1.0, 10))
    
    print("Training size | Train score | Val score")
    for size, tr, val in zip(train_sizes, train_scores.mean(axis=1),
                                val_scores.mean(axis=1)):
        print(f"  {size:11d} | {tr:.4f}      | {val:.4f}")
    
    gap = train_scores.mean() - val_scores.mean()
    print(f"\nGap: {gap:.4f}")
    print("Overfitting!" if gap > 0.1 else "Good fit")

# Compare overfitting (deep tree) vs regularized (limited depth)
from sklearn.tree import DecisionTreeClassifier
data = load_breast_cancer()

print("=== Deep Tree (likely overfit) ===")
plot_learning_curve(DecisionTreeClassifier(max_depth=None), data.data, data.target)

print("\n=== Shallow Tree (regularized) ===")
plot_learning_curve(DecisionTreeClassifier(max_depth=3), data.data, data.target)

Scikit-learn Pipelines & Feature Engineering

Python
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
import pandas as pd, numpy as np

# Simulated mixed-type dataset
df = pd.DataFrame({
    'age': [25, 32, np.nan, 45, 28, 52, 38],
    'income': [30000, 55000, 42000, np.nan, 48000, 72000, 60000],
    'education': ['BS','MS','BS','PhD','BS','MS','PhD'],
    'target': [0,1,0,1,0,1,1]
})

numeric_features = ['age', 'income']
categorical_features = ['education']

# Column-specific preprocessing
preprocessor = ColumnTransformer([
    ('num', Pipeline([
        ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())
    ]), numeric_features),
    ('cat', Pipeline([
        ('imputer', SimpleImputer(strategy='most_frequent')),
        ('encoder', OneHotEncoder(handle_unknown='ignore'))
    ]), categorical_features)
])

# Full pipeline: preprocess + model
pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(n_estimators=100))
])

X = df.drop('target', axis=1)
y = df['target']
pipeline.fit(X, y)
print(f"Pipeline trained successfully!")
print(f"Prediction: {pipeline.predict(X[:2])}")

Python
import pandas as pd
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score

df = pd.read_csv('https://raw.githubusercontent.com/datasciencedojo/datasets/master/titanic.csv')
features = ['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']
X, y = df[features], df['Survived']

preprocessor = ColumnTransformer([
    ('num', Pipeline([
        ('imp', SimpleImputer(strategy='median')),
        ('scl', StandardScaler())
    ]), ['Age','SibSp','Parch','Fare']),
    ('cat', Pipeline([
        ('imp', SimpleImputer(strategy='most_frequent')),
        ('enc', OneHotEncoder(handle_unknown='ignore'))
    ]), ['Pclass','Sex','Embarked'])
])

pipe = Pipeline([('pre', preprocessor), ('clf', GradientBoostingClassifier())])
scores = cross_val_score(pipe, X, y, cv=5, scoring='accuracy')
print(f"Titanic CV Accuracy: {scores.mean():.2%} ± {scores.std():.2%}")

from sklearn.base import BaseEstimator, TransformerMixin
class LogTransformer(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None): return self
    def transform(self, X): return np.log1p(X)

Hyperparameter Tuning & MLflow

GridSearch vs RandomizedSearch

Python
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
from scipy.stats import randint, uniform

data = load_breast_cancer()
X, y = data.data, data.target

# GridSearchCV — exhaustive (try ALL combinations)
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [3, 5, 7, None],
    'min_samples_split': [2, 5, 10]
}

grid = GridSearchCV(RandomForestClassifier(random_state=42),
                    param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid.fit(X, y)
print(f"Best params: {grid.best_params_}")
print(f"Best score:  {grid.best_score_:.4f}")

# RandomizedSearchCV — samples N random combos (faster for large spaces)
param_dist = {
    'n_estimators': randint(50, 500),
    'max_depth': [3, 5, 7, 10, None],
    'min_samples_split': randint(2, 20),
    'max_features': uniform(0.1, 0.9)
}

random_search = RandomizedSearchCV(
    RandomForestClassifier(random_state=42), param_dist,
    n_iter=50, cv=5, scoring='accuracy', n_jobs=-1)
random_search.fit(X, y)
print(f"\nRandom best: {random_search.best_params_}")
print(f"Random score: {random_search.best_score_:.4f}")

MLflow — Experiment Tracking

Python
# pip install mlflow
import mlflow
import mlflow.sklearn

mlflow.set_experiment("breast_cancer_classification")

for n_est in [50, 100, 200]:
    for max_d in [3, 5, 7]:
        with mlflow.start_run(run_name=f"RF_{n_est}_{max_d}"):
            # Train
            rf = RandomForestClassifier(n_estimators=n_est, max_depth=max_d)
            scores = cross_val_score(rf, X, y, cv=5)

            # Log parameters and metrics
            mlflow.log_param("n_estimators", n_est)
            mlflow.log_param("max_depth", max_d)
            mlflow.log_metric("cv_accuracy_mean", scores.mean())
            mlflow.log_metric("cv_accuracy_std", scores.std())

            # Log model
            rf.fit(X, y)
            mlflow.sklearn.log_model(rf, "model")

print("All experiments logged! Run 'mlflow ui' to view.")

# Save model
mlflow.sklearn.log_model(best_model, "model")

# Load and serve
loaded = mlflow.sklearn.load_model("runs:/<run_id>/model")
prediction = loaded.predict(new_data)

# Or serve as REST API:
# mlflow models serve -m runs:/<run_id>/model --port 5000