SVMs, KNN & Naive Bayes

import numpy as np
from collections import Counter

class KNNClassifier:
    """
    K-Nearest Neighbors Classifier — built from scratch.
    
    Supports multiple distance metrics:
      - 'euclidean': L2 norm
      - 'manhattan': L1 norm
      - 'minkowski': Lp norm (specify p)
      - 'cosine':    1 - cosine similarity
    
    Usage:
        knn = KNNClassifier(k=5, metric='euclidean')
        knn.fit(X_train, y_train)
        predictions = knn.predict(X_test)
    """
    
    def __init__(self, k=5, metric='euclidean', p=2, weighted=False):
        self.k = k
        self.metric = metric
        self.p = p  # For Minkowski distance
        self.weighted = weighted  # Use distance-weighted voting
        self.X_train = None
        self.y_train = None
    
    def fit(self, X, y):
        """Store training data (lazy learning — no actual training!)."""
        self.X_train = np.array(X, dtype=np.float64)
        self.y_train = np.array(y)
        return self
    
    def _compute_distance(self, x1, x2):
        """Compute distance between two vectors."""
        if self.metric == 'euclidean':
            return np.sqrt(np.sum((x1 - x2) ** 2))
        elif self.metric == 'manhattan':
            return np.sum(np.abs(x1 - x2))
        elif self.metric == 'minkowski':
            return np.sum(np.abs(x1 - x2) ** self.p) ** (1 / self.p)
        elif self.metric == 'cosine':
            dot = np.dot(x1, x2)
            norm1 = np.linalg.norm(x1)
            norm2 = np.linalg.norm(x2)
            if norm1 == 0 or norm2 == 0:
                return 1.0
            return 1.0 - dot / (norm1 * norm2)
        else:
            raise ValueError(f"Unknown metric: {self.metric}")
    
    def _predict_single(self, x):
        """Predict class for a single query point."""
        # Compute distances to all training points
        distances = np.array([
            self._compute_distance(x, x_train)
            for x_train in self.X_train
        ])
        
        # Get indices of K nearest neighbors
        k_indices = np.argsort(distances)[:self.k]
        k_labels = self.y_train[k_indices]
        k_distances = distances[k_indices]
        
        if self.weighted:
            # Distance-weighted voting
            weights = {}
            for label, dist in zip(k_labels, k_distances):
                w = 1.0 / (dist + 1e-8)  # Avoid division by zero
                weights[label] = weights.get(label, 0) + w
            return max(weights, key=weights.get)
        else:
            # Simple majority voting
            counter = Counter(k_labels)
            return counter.most_common(1)[0][0]
    
    def predict(self, X):
        """Predict classes for multiple query points."""
        X = np.array(X, dtype=np.float64)
        return np.array([self._predict_single(x) for x in X])
    
    def score(self, X, y):
        """Compute classification accuracy."""
        predictions = self.predict(X)
        return np.mean(predictions == np.array(y))


# ─── Demo ─────────────────────────────────────────────────
if __name__ == "__main__":
    from sklearn.datasets import load_iris
    from sklearn.model_selection import train_test_split
    
    # Load Iris dataset
    iris = load_iris()
    X_train, X_test, y_train, y_test = train_test_split(
        iris.data, iris.target, test_size=0.3, random_state=42
    )
    
    # Test different K values
    for k in [1, 3, 5, 7, 11]:
        knn = KNNClassifier(k=k, metric='euclidean')
        knn.fit(X_train, y_train)
        accuracy = knn.score(X_test, y_test)
        print(f"K={k:2d} | Accuracy: {accuracy:.4f}")
    
    # Test different metrics
    print("\n--- Distance Metrics Comparison ---")
    for metric in ['euclidean', 'manhattan', 'cosine']:
        knn = KNNClassifier(k=5, metric=metric)
        knn.fit(X_train, y_train)
        accuracy = knn.score(X_test, y_test)
        print(f"Metric: {metric:10s} | Accuracy: {accuracy:.4f}")
    
    # Weighted vs Unweighted
    print("\n--- Weighted KNN ---")
    knn_unweighted = KNNClassifier(k=5, weighted=False)
    knn_weighted = KNNClassifier(k=5, weighted=True)
    knn_unweighted.fit(X_train, y_train)
    knn_weighted.fit(X_train, y_train)
    print(f"Unweighted K=5: {knn_unweighted.score(X_test, y_test):.4f}")
    print(f"Weighted   K=5: {knn_weighted.score(X_test, y_test):.4f}")

import numpy as np

class GaussianNaiveBayes:
    """
    Gaussian Naive Bayes Classifier — built from scratch.
    
    Assumes each feature follows a Gaussian distribution
    within each class: P(xⱼ|y=c) = N(μⱼc, σ²ⱼc)
    
    Uses log probabilities to avoid numerical underflow.
    """
    
    def __init__(self):
        self.classes = None
        self.priors = {}       # P(y = c) for each class
        self.means = {}        # μⱼc for each class and feature
        self.variances = {}    # σ²ⱼc for each class and feature
    
    def fit(self, X, y):
        """
        Learn class priors and per-class Gaussian parameters.
        
        Parameters:
            X: np.ndarray of shape (n_samples, n_features)
            y: np.ndarray of shape (n_samples,)
        """
        X = np.array(X, dtype=np.float64)
        y = np.array(y)
        n_samples = len(y)
        self.classes = np.unique(y)
        
        for c in self.classes:
            X_c = X[y == c]
            self.priors[c] = len(X_c) / n_samples
            self.means[c] = X_c.mean(axis=0)
            # Add small epsilon to variance for numerical stability
            self.variances[c] = X_c.var(axis=0) + 1e-9
        
        return self
    
    def _log_gaussian_pdf(self, x, mean, var):
        """
        Log of Gaussian probability density function.
        
        log P(x|μ,σ²) = -½ log(2πσ²) - (x-μ)²/(2σ²)
        """
        return -0.5 * np.log(2 * np.pi * var) - (x - mean) ** 2 / (2 * var)
    
    def _log_posterior(self, x):
        """
        Compute log posterior for each class.
        
        log P(y=c|x) ∝ log P(y=c) + Σⱼ log P(xⱼ|y=c)
        """
        log_posts = {}
        for c in self.classes:
            # Log prior
            log_prior = np.log(self.priors[c])
            # Sum of log-likelihoods (naive independence)
            log_likelihood = np.sum(
                self._log_gaussian_pdf(x, self.means[c], self.variances[c])
            )
            log_posts[c] = log_prior + log_likelihood
        return log_posts
    
    def predict(self, X):
        """Predict class labels for samples in X."""
        X = np.array(X, dtype=np.float64)
        predictions = []
        for x in X:
            log_posts = self._log_posterior(x)
            predicted_class = max(log_posts, key=log_posts.get)
            predictions.append(predicted_class)
        return np.array(predictions)
    
    def predict_proba(self, X):
        """
        Predict class probabilities using softmax on log posteriors.
        """
        X = np.array(X, dtype=np.float64)
        all_probs = []
        for x in X:
            log_posts = self._log_posterior(x)
            log_values = np.array([log_posts[c] for c in self.classes])
            # Softmax for numerical stability
            log_values -= np.max(log_values)
            probs = np.exp(log_values)
            probs /= probs.sum()
            all_probs.append(probs)
        return np.array(all_probs)
    
    def score(self, X, y):
        """Compute classification accuracy."""
        return np.mean(self.predict(X) == np.array(y))


class MultinomialNaiveBayes:
    """
    Multinomial Naive Bayes — for text classification (word counts).
    Includes Laplace smoothing.
    """
    
    def __init__(self, alpha=1.0):
        self.alpha = alpha  # Laplace smoothing parameter
        self.classes = None
        self.log_priors = {}
        self.log_likelihoods = {}
    
    def fit(self, X, y):
        """
        X: (n_samples, n_features) — word count matrix
        y: (n_samples,) — class labels
        """
        X = np.array(X, dtype=np.float64)
        y = np.array(y)
        n_samples, n_features = X.shape
        self.classes = np.unique(y)
        
        for c in self.classes:
            X_c = X[y == c]
            # Log prior
            self.log_priors[c] = np.log(len(X_c) / n_samples)
            # Word counts per class + smoothing
            word_counts = X_c.sum(axis=0) + self.alpha
            total_count = word_counts.sum()
            # Log likelihood for each word
            self.log_likelihoods[c] = np.log(word_counts / total_count)
        
        return self
    
    def predict(self, X):
        X = np.array(X, dtype=np.float64)
        predictions = []
        for x in X:
            log_posts = {}
            for c in self.classes:
                log_posts[c] = self.log_priors[c] + np.sum(
                    x * self.log_likelihoods[c]
                )
            predictions.append(max(log_posts, key=log_posts.get))
        return np.array(predictions)
    
    def score(self, X, y):
        return np.mean(self.predict(X) == np.array(y))


# ─── Demo ─────────────────────────────────────────────────
if __name__ == "__main__":
    from sklearn.datasets import load_iris
    from sklearn.model_selection import train_test_split
    
    iris = load_iris()
    X_train, X_test, y_train, y_test = train_test_split(
        iris.data, iris.target, test_size=0.3, random_state=42
    )
    
    # Gaussian NB
    gnb = GaussianNaiveBayes()
    gnb.fit(X_train, y_train)
    print(f"Gaussian NB Accuracy: {gnb.score(X_test, y_test):.4f}")
    print(f"Sample probabilities:\n{gnb.predict_proba(X_test[:3])}")
    
    # Multinomial NB (on non-negative data)
    X_pos = iris.data  # Already non-negative
    X_tr, X_te, y_tr, y_te = train_test_split(
        X_pos, iris.target, test_size=0.3, random_state=42
    )
    mnb = MultinomialNaiveBayes(alpha=1.0)
    mnb.fit(X_tr, y_tr)
    print(f"\nMultinomial NB Accuracy: {mnb.score(X_te, y_te):.4f}")

import numpy as np

class SimpleSVM:
    """
    Simplified Support Vector Machine using gradient descent.
    
    Minimizes the hinge loss: L = ½‖w‖² + C Σ max(0, 1 - yᵢ(w·xᵢ + b))
    
    This is a primal-form SVM using sub-gradient descent.
    For production, use sklearn's SVC which uses libsvm (SMO algorithm).
    """
    
    def __init__(self, C=1.0, learning_rate=0.001, n_iters=1000):
        self.C = C
        self.lr = learning_rate
        self.n_iters = n_iters
        self.w = None
        self.b = None
        self.losses = []
    
    def fit(self, X, y):
        """
        Train SVM using sub-gradient descent on hinge loss.
        
        Parameters:
            X: (n_samples, n_features) training features
            y: (n_samples,) labels in {-1, +1}
        """
        X = np.array(X, dtype=np.float64)
        y = np.array(y, dtype=np.float64)
        
        # Ensure labels are -1 and +1
        unique_labels = np.unique(y)
        if set(unique_labels) != {-1.0, 1.0}:
            # Convert 0/1 to -1/+1
            y = np.where(y <= 0, -1.0, 1.0)
        
        n_samples, n_features = X.shape
        
        # Initialize weights
        self.w = np.zeros(n_features)
        self.b = 0.0
        
        for iteration in range(self.n_iters):
            total_loss = 0
            
            for i in range(n_samples):
                # Functional margin
                margin = y[i] * (np.dot(self.w, X[i]) + self.b)
                
                if margin >= 1:
                    # Correctly classified with sufficient margin
                    # Gradient: just the regularization term
                    self.w -= self.lr * self.w  # ∂(½‖w‖²)/∂w = w
                else:
                    # Misclassified or within margin
                    # Gradient: regularization + hinge loss gradient
                    self.w -= self.lr * (self.w - self.C * y[i] * X[i])
                    self.b -= self.lr * (-self.C * y[i])
                    total_loss += 1 - margin
            
            # Total loss = regularization + C * hinge
            reg_loss = 0.5 * np.dot(self.w, self.w)
            self.losses.append(reg_loss + self.C * total_loss)
        
        return self
    
    def predict(self, X):
        """Predict class labels (+1 or -1)."""
        X = np.array(X, dtype=np.float64)
        return np.sign(np.dot(X, self.w) + self.b)
    
    def decision_function(self, X):
        """Return raw decision values (distance from hyperplane)."""
        X = np.array(X, dtype=np.float64)
        return np.dot(X, self.w) + self.b
    
    def score(self, X, y):
        """Compute classification accuracy."""
        y = np.array(y, dtype=np.float64)
        y = np.where(y <= 0, -1.0, 1.0)
        predictions = self.predict(X)
        return np.mean(predictions == y)
    
    def get_margin(self):
        """Return the margin width 2/‖w‖."""
        w_norm = np.linalg.norm(self.w)
        if w_norm == 0:
            return float('inf')
        return 2.0 / w_norm


# ─── Demo ─────────────────────────────────────────────────
if __name__ == "__main__":
    from sklearn.datasets import make_classification
    from sklearn.model_selection import train_test_split
    from sklearn.preprocessing import StandardScaler
    
    # Generate linearly separable data
    X, y = make_classification(
        n_samples=200, n_features=2, n_redundant=0,
        n_informative=2, n_clusters_per_class=1, random_state=42
    )
    y = np.where(y == 0, -1, 1)  # Convert to -1/+1
    
    # Scale features (important for SVM!)
    scaler = StandardScaler()
    X = scaler.fit_transform(X)
    
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.3, random_state=42
    )
    
    # Train SVM with different C values
    for C in [0.01, 0.1, 1.0, 10.0]:
        svm = SimpleSVM(C=C, learning_rate=0.001, n_iters=1000)
        svm.fit(X_train, y_train)
        train_acc = svm.score(X_train, y_train)
        test_acc = svm.score(X_test, y_test)
        margin = svm.get_margin()
        print(f"C={C:5.2f} | Train Acc: {train_acc:.4f} | "
              f"Test Acc: {test_acc:.4f} | Margin: {margin:.4f}")
    
    print(f"\nWeight vector w: {svm.w}")
    print(f"Bias b: {svm.b:.4f}")
    print(f"Margin width: {svm.get_margin():.4f}")

import tensorflow as tf
import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# ─── Linear SVM using TensorFlow (Hinge Loss) ─────────────────

class TFLinearSVM(tf.Module):
    """
    Linear SVM implemented in TensorFlow using hinge loss.
    Useful for GPU acceleration on large datasets.
    """
    
    def __init__(self, n_features, C=1.0, name='LinearSVM'):
        super().__init__(name=name)
        self.C = C
        self.w = tf.Variable(
            tf.random.normal([n_features, 1], stddev=0.01),
            name='weights'
        )
        self.b = tf.Variable(tf.zeros([1]), name='bias')
    
    def __call__(self, X):
        return tf.matmul(X, self.w) + self.b
    
    def hinge_loss(self, X, y):
        """
        Compute SVM objective: ½‖w‖² + C × Σ max(0, 1 - y(wx+b))
        """
        y = tf.cast(tf.reshape(y, [-1, 1]), tf.float32)
        logits = self(X)
        
        # Hinge loss: max(0, 1 - y * f(x))
        hinge = tf.reduce_mean(tf.maximum(0.0, 1.0 - y * logits))
        
        # L2 regularization: ½‖w‖²
        reg = 0.5 * tf.reduce_sum(tf.square(self.w))
        
        return reg + self.C * hinge


def train_tf_svm():
    """End-to-end TensorFlow SVM training pipeline."""
    
    # Generate data
    X, y = make_classification(
        n_samples=1000, n_features=10, n_informative=5,
        n_redundant=2, random_state=42
    )
    y = np.where(y == 0, -1, 1).astype(np.float32)
    
    # Scale features
    scaler = StandardScaler()
    X = scaler.fit_transform(X).astype(np.float32)
    
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    # Convert to TF tensors
    X_train_tf = tf.constant(X_train)
    y_train_tf = tf.constant(y_train)
    X_test_tf = tf.constant(X_test)
    y_test_tf = tf.constant(y_test)
    
    # Build model
    model = TFLinearSVM(n_features=10, C=1.0)
    optimizer = tf.optimizers.Adam(learning_rate=0.01)
    
    # Training loop
    print("Training TensorFlow SVM...")
    for epoch in range(200):
        with tf.GradientTape() as tape:
            loss = model.hinge_loss(X_train_tf, y_train_tf)
        
        gradients = tape.gradient(loss, model.trainable_variables)
        optimizer.apply_gradients(zip(gradients, model.trainable_variables))
        
        if (epoch + 1) % 50 == 0:
            # Compute accuracy
            train_pred = tf.sign(model(X_train_tf))
            train_acc = tf.reduce_mean(
                tf.cast(tf.equal(train_pred, tf.reshape(y_train_tf, [-1, 1])), 
                tf.float32)
            )
            test_pred = tf.sign(model(X_test_tf))
            test_acc = tf.reduce_mean(
                tf.cast(tf.equal(test_pred, tf.reshape(y_test_tf, [-1, 1])),
                tf.float32)
            )
            print(f"Epoch {epoch+1:3d} | Loss: {loss:.4f} | "
                  f"Train Acc: {train_acc:.4f} | Test Acc: {test_acc:.4f}")
    
    # Compute margin
    w_norm = tf.norm(model.w).numpy()
    margin = 2.0 / w_norm
    print(f"\nFinal margin width: {margin:.4f}")
    print(f"Weight vector norm: {w_norm:.4f}")


# ─── KNN with TensorFlow (GPU-accelerated distances) ──────────

def tf_knn_predict(X_train, y_train, X_test, k=5):
    """
    KNN using TensorFlow for GPU-accelerated distance computation.
    Useful when training set is very large.
    """
    X_train = tf.constant(X_train, dtype=tf.float32)
    X_test = tf.constant(X_test, dtype=tf.float32)
    
    # Compute pairwise Euclidean distances using broadcasting
    # ||a - b||² = ||a||² + ||b||² - 2a·b
    train_sq = tf.reduce_sum(tf.square(X_train), axis=1, keepdims=True)
    test_sq = tf.reduce_sum(tf.square(X_test), axis=1, keepdims=True)
    cross = tf.matmul(X_test, X_train, transpose_b=True)
    
    distances = tf.sqrt(
        tf.maximum(test_sq + tf.transpose(train_sq) - 2 * cross, 0.0)
    )
    
    # Get K nearest neighbors
    _, top_k_indices = tf.math.top_k(-distances, k=k)
    
    # Gather labels
    predictions = []
    for i in range(len(X_test)):
        neighbor_labels = tf.gather(y_train, top_k_indices[i])
        # Majority vote
        unique, _, counts = tf.unique_with_counts(neighbor_labels)
        majority_idx = tf.argmax(counts)
        predictions.append(unique[majority_idx].numpy())
    
    return np.array(predictions)


if __name__ == "__main__":
    train_tf_svm()

import numpy as np
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.model_selection import (
    train_test_split, GridSearchCV, cross_val_score
)
from sklearn.metrics import (
    classification_report, confusion_matrix, accuracy_score
)
from sklearn.datasets import load_iris, load_digits
import warnings
warnings.filterwarnings('ignore')


# ═══════════════════════════════════════════════════════════
# 1. SVM Pipeline with Kernel Comparison
# ═══════════════════════════════════════════════════════════

def svm_kernel_comparison():
    """Compare SVM with different kernels on Iris dataset."""
    print("=" * 60)
    print("SVM KERNEL COMPARISON — Iris Dataset")
    print("=" * 60)
    
    iris = load_iris()
    X_train, X_test, y_train, y_test = train_test_split(
        iris.data, iris.target, test_size=0.3, random_state=42
    )
    
    kernels = {
        'Linear':     SVC(kernel='linear', C=1.0),
        'Polynomial': SVC(kernel='poly', degree=3, C=1.0),
        'RBF':        SVC(kernel='rbf', gamma='scale', C=1.0),
        'Sigmoid':    SVC(kernel='sigmoid', C=1.0),
    }
    
    for name, model in kernels.items():
        pipeline = Pipeline([
            ('scaler', StandardScaler()),
            ('svm', model)
        ])
        pipeline.fit(X_train, y_train)
        train_acc = pipeline.score(X_train, y_train)
        test_acc = pipeline.score(X_test, y_test)
        
        # Number of support vectors
        svm_model = pipeline.named_steps['svm']
        n_sv = svm_model.n_support_.sum()
        
        print(f"\n{name:12s} Kernel:")
        print(f"  Train Accuracy: {train_acc:.4f}")
        print(f"  Test Accuracy:  {test_acc:.4f}")
        print(f"  Support Vectors: {n_sv} / {len(y_train)} "
              f"({100*n_sv/len(y_train):.1f}%)")


# ═══════════════════════════════════════════════════════════
# 2. SVM Hyperparameter Tuning with GridSearchCV
# ═══════════════════════════════════════════════════════════

def svm_grid_search():
    """Find optimal C and gamma for RBF SVM."""
    print("\n" + "=" * 60)
    print("SVM GRID SEARCH — Optimal C and Gamma")
    print("=" * 60)
    
    digits = load_digits()
    X_train, X_test, y_train, y_test = train_test_split(
        digits.data, digits.target, test_size=0.2, random_state=42
    )
    
    pipeline = Pipeline([
        ('scaler', StandardScaler()),
        ('svm', SVC())
    ])
    
    param_grid = {
        'svm__C': [0.1, 1, 10, 100],
        'svm__gamma': ['scale', 'auto', 0.001, 0.01],
        'svm__kernel': ['rbf', 'linear']
    }
    
    grid = GridSearchCV(
        pipeline, param_grid, cv=5,
        scoring='accuracy', n_jobs=-1, verbose=0
    )
    grid.fit(X_train, y_train)
    
    print(f"Best Parameters: {grid.best_params_}")
    print(f"Best CV Accuracy: {grid.best_score_:.4f}")
    print(f"Test Accuracy:    {grid.score(X_test, y_test):.4f}")
    
    # Classification report
    y_pred = grid.predict(X_test)
    print(f"\nClassification Report:\n{classification_report(y_pred=y_pred, y_true=y_test)}")


# ═══════════════════════════════════════════════════════════
# 3. KNN Pipeline with K Selection
# ═══════════════════════════════════════════════════════════

def knn_k_selection():
    """Find optimal K using cross-validation."""
    print("\n" + "=" * 60)
    print("KNN — Optimal K Selection")
    print("=" * 60)
    
    iris = load_iris()
    X_train, X_test, y_train, y_test = train_test_split(
        iris.data, iris.target, test_size=0.3, random_state=42
    )
    
    # Test K from 1 to 25
    k_values = range(1, 26)
    cv_scores = []
    
    for k in k_values:
        knn = Pipeline([
            ('scaler', StandardScaler()),
            ('knn', KNeighborsClassifier(n_neighbors=k))
        ])
        scores = cross_val_score(knn, X_train, y_train, cv=5, scoring='accuracy')
        cv_scores.append(scores.mean())
        if k % 5 == 0 or k == 1:
            print(f"K={k:2d} | CV Accuracy: {scores.mean():.4f} ± {scores.std():.4f}")
    
    best_k = k_values[np.argmax(cv_scores)]
    print(f"\nBest K = {best_k} with CV accuracy = {max(cv_scores):.4f}")
    
    # Final model with best K
    final_knn = Pipeline([
        ('scaler', StandardScaler()),
        ('knn', KNeighborsClassifier(n_neighbors=best_k, weights='distance'))
    ])
    final_knn.fit(X_train, y_train)
    print(f"Test Accuracy with K={best_k}: {final_knn.score(X_test, y_test):.4f}")


# ═══════════════════════════════════════════════════════════
# 4. Naive Bayes — All Three Variants
# ═══════════════════════════════════════════════════════════

def naive_bayes_variants():
    """Compare Gaussian, Multinomial, and Bernoulli NB."""
    print("\n" + "=" * 60)
    print("NAIVE BAYES — Three Variants")
    print("=" * 60)
    
    iris = load_iris()
    X_train, X_test, y_train, y_test = train_test_split(
        iris.data, iris.target, test_size=0.3, random_state=42
    )
    
    # Gaussian NB — works with continuous features
    gnb = GaussianNB()
    gnb.fit(X_train, y_train)
    print(f"\nGaussian NB:")
    print(f"  Accuracy: {gnb.score(X_test, y_test):.4f}")
    print(f"  Class priors: {gnb.class_prior_}")
    print(f"  Feature means (class 0): {gnb.theta_[0].round(3)}")
    
    # Multinomial NB — needs non-negative features
    scaler = MinMaxScaler()  # Scale to [0, 1]
    X_train_pos = scaler.fit_transform(X_train)
    X_test_pos = scaler.transform(X_test)
    
    mnb = MultinomialNB(alpha=1.0)  # alpha = Laplace smoothing
    mnb.fit(X_train_pos, y_train)
    print(f"\nMultinomial NB (alpha=1.0):")
    print(f"  Accuracy: {mnb.score(X_test_pos, y_test):.4f}")
    
    # Bernoulli NB — works with binary features
    X_train_bin = (X_train > X_train.mean(axis=0)).astype(int)
    X_test_bin = (X_test > X_train.mean(axis=0)).astype(int)
    
    bnb = BernoulliNB(alpha=1.0)
    bnb.fit(X_train_bin, y_train)
    print(f"\nBernoulli NB:")
    print(f"  Accuracy: {bnb.score(X_test_bin, y_test):.4f}")


# ═══════════════════════════════════════════════════════════
# 5. Head-to-Head Comparison
# ═══════════════════════════════════════════════════════════

def head_to_head():
    """Compare SVM, KNN, NB on the same dataset."""
    print("\n" + "=" * 60)
    print("HEAD-TO-HEAD: SVM vs KNN vs Naive Bayes")
    print("=" * 60)
    
    digits = load_digits()
    X_train, X_test, y_train, y_test = train_test_split(
        digits.data, digits.target, test_size=0.2, random_state=42
    )
    
    models = {
        'SVM (RBF)': Pipeline([
            ('scaler', StandardScaler()),
            ('clf', SVC(kernel='rbf', C=10, gamma='scale'))
        ]),
        'KNN (K=5)': Pipeline([
            ('scaler', StandardScaler()),
            ('clf', KNeighborsClassifier(n_neighbors=5, weights='distance'))
        ]),
        'Gaussian NB': Pipeline([
            ('scaler', StandardScaler()),
            ('clf', GaussianNB())
        ]),
    }
    
    import time
    
    print(f"\n{'Model':<15} {'Train Acc':<12} {'Test Acc':<12} "
          f"{'Train Time':<12} {'Pred Time'}")
    print("-" * 65)
    
    for name, model in models.items():
        # Training time
        t0 = time.time()
        model.fit(X_train, y_train)
        train_time = time.time() - t0
        
        # Prediction time
        t0 = time.time()
        y_pred = model.predict(X_test)
        pred_time = time.time() - t0
        
        train_acc = model.score(X_train, y_train)
        test_acc = accuracy_score(y_test, y_pred)
        
        print(f"{name:<15} {train_acc:<12.4f} {test_acc:<12.4f} "
              f"{train_time:<12.4f} {pred_time:.4f}s")


# ─── Run all experiments ───────────────────────────────────
if __name__ == "__main__":
    svm_kernel_comparison()
    svm_grid_search()
    knn_k_selection()
    naive_bayes_variants()
    head_to_head()

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
import numpy as np

# Simulated IRCTC complaint data
complaints = [
    "My refund has not been processed for cancelled train",
    "Refund pending for 3 weeks after ticket cancellation",
    "I want my money back for cancelled journey",
    "Unable to book ticket, payment deducted but no confirmation",
    "Booking failed but amount debited from bank account",
    "Double booking happened, charged twice",
    "Train was delayed by 6 hours, no announcement",
    "Rajdhani Express arrived 4 hours late",
    "My train has been rescheduled without any notification",
    "Food quality was terrible in Duronto Express",
    "Got stale food in pantry car, fell sick after eating",
    "No vegetarian options available in catering service",
    "Toilet was extremely dirty and unhygienic",
    "AC was not working in my coach, very hot",
    "Cockroaches found in the sleeper berth",
    "TTE was very rude and misbehaved with passengers",
    "Railway staff refused to help elderly passenger",
    "Conductor was not cooperating during ticket checking",
    "IRCTC website keeps crashing during Tatkal booking",
    "App not loading, technical error during payment",
    "OTP not received for login, can't access account",
]

categories = [
    "Refund", "Refund", "Refund",
    "Booking Error", "Booking Error", "Booking Error",
    "Train Delay", "Train Delay", "Train Delay",
    "Food Quality", "Food Quality", "Food Quality",
    "Cleanliness", "Cleanliness", "Cleanliness",
    "Staff Behavior", "Staff Behavior", "Staff Behavior",
    "Technical Issue", "Technical Issue", "Technical Issue",
]

# Build classification pipeline
pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        max_features=500,
        ngram_range=(1, 2),     # Unigrams + bigrams
        stop_words='english'
    )),
    ('nb', MultinomialNB(alpha=1.0))  # Laplace smoothing
])

# Train and evaluate
pipeline.fit(complaints, categories)

# Test with new complaints
test_complaints = [
    "When will I get refund for my cancelled ticket?",
    "Train was 5 hours late from Delhi",
    "The biryani in pantry was cold and tasteless",
    "Website crashed during Tatkal booking window",
    "The TTE shouted at me for no reason",
]

predictions = pipeline.predict(test_complaints)
for complaint, category in zip(test_complaints, predictions):
    print(f"  Complaint: {complaint}")
    print(f"  Category:  {category}\n")

# Cross-validation (limited data, so results vary)
scores = cross_val_score(pipeline, complaints, categories, cv=3)
print(f"Cross-validation accuracy: {scores.mean():.3f} ± {scores.std():.3f}")

from sklearn.svm import LinearSVC
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report
import numpy as np

# Simulated Flipkart review data
reviews = [
    "Excellent product! Very happy with the quality",
    "Best phone I have ever purchased, amazing camera",
    "Great value for money, highly recommend",
    "Product delivered fast, packaging was perfect",
    "Love the color and design, looks premium",
    "Worst product ever, broke within 2 days",
    "Terrible quality, waste of money",
    "Do not buy, cheap material used",
    "Product is defective, screen stopped working",
    "Very disappointed with the purchase",
    "Product is okay, nothing special",
    "Average quality, expected better at this price",
    "Decent product but delivery was late",
    "Good features but battery life is average",
    "Fair product, meets basic expectations",
]

sentiments = [
    "Positive", "Positive", "Positive", "Positive", "Positive",
    "Negative", "Negative", "Negative", "Negative", "Negative",
    "Neutral", "Neutral", "Neutral", "Neutral", "Neutral",
]

# SVM Pipeline with TF-IDF
svm_pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        max_features=1000,
        ngram_range=(1, 2),
        sublinear_tf=True     # Apply log normalization
    )),
    ('svm', LinearSVC(
        C=1.0,
        class_weight='balanced',  # Handle class imbalance
        max_iter=10000
    ))
])

svm_pipeline.fit(reviews, sentiments)

# Classify new reviews
new_reviews = [
    "This mobile phone is absolutely fantastic!",
    "Pathetic product, returning immediately",
    "It's an okay laptop for basic work",
    "Camera quality is superb, loved it",
    "Not worth the price, very cheap build quality",
]

predictions = svm_pipeline.predict(new_reviews)
# Get decision function scores for confidence
decision_scores = svm_pipeline.decision_function(new_reviews)

print("Flipkart Review Sentiment Analysis (SVM)")
print("=" * 55)
for review, pred in zip(new_reviews, predictions):
    print(f"  Review:    {review}")
    print(f"  Sentiment: {pred}\n")

from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import confusion_matrix, classification_report
import numpy as np

# Using sklearn digits as proxy (8x8 pixel images, 0-9)
# In practice, use ISI Kolkata's Devanagari digit dataset
digits = load_digits()

print(f"Dataset: {digits.data.shape[0]} images, "
      f"{digits.data.shape[1]} features (8x8 pixels)")
print(f"Classes: {np.unique(digits.target)}")

X_train, X_test, y_train, y_test = train_test_split(
    digits.data, digits.target, test_size=0.2, random_state=42
)

# Find optimal K
print("\n--- K Selection via Cross-Validation ---")
best_k, best_score = 1, 0
for k in [1, 3, 5, 7, 9, 11, 15]:
    knn = Pipeline([
        ('scaler', StandardScaler()),
        ('knn', KNeighborsClassifier(n_neighbors=k, weights='distance'))
    ])
    scores = cross_val_score(knn, X_train, y_train, cv=5)
    mean_score = scores.mean()
    print(f"  K={k:2d}: {mean_score:.4f} ± {scores.std():.4f}")
    if mean_score > best_score:
        best_k, best_score = k, mean_score

print(f"\nBest K = {best_k}")

# Train final model
final_model = Pipeline([
    ('scaler', StandardScaler()),
    ('knn', KNeighborsClassifier(n_neighbors=best_k, weights='distance'))
])
final_model.fit(X_train, y_train)

y_pred = final_model.predict(X_test)
print(f"\nTest Accuracy: {(y_pred == y_test).mean():.4f}")
print(f"\nClassification Report:\n{classification_report(y_test, y_pred)}")

# Show confusion matrix
cm = confusion_matrix(y_test, y_pred)
print("Confusion Matrix:")
print(cm)

# Which digits are most confused?
errors = []
for i in range(10):
    for j in range(10):
        if i != j and cm[i][j] > 0:
            errors.append((i, j, cm[i][j]))
errors.sort(key=lambda x: -x[2])
print("\nMost confused digit pairs:")
for true, pred, count in errors[:5]:
    print(f"  Digit {true} misclassified as {pred}: {count} times")

"""
MINI PROJECT 1: Spam Email Classifier
======================================
Compare NB vs SVM on email spam detection.
Uses sklearn's 20 newsgroups as a proxy for email data.
"""

import numpy as np
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.metrics import (
    classification_report, confusion_matrix,
    precision_recall_fscore_support
)
import time

# ─── Step 1: Load Data ────────────────────────────────────
# Using 2 categories as spam/ham proxy
categories = ['rec.sport.hockey', 'sci.med']  # Very different topics
data_train = fetch_20newsgroups(
    subset='train', categories=categories, 
    remove=('headers', 'footers', 'quotes')
)
data_test = fetch_20newsgroups(
    subset='test', categories=categories,
    remove=('headers', 'footers', 'quotes')
)

print(f"Training samples: {len(data_train.data)}")
print(f"Test samples:     {len(data_test.data)}")
print(f"Categories:       {data_train.target_names}")

# ─── Step 2: Build Pipelines ──────────────────────────────
nb_pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        max_features=10000,
        ngram_range=(1, 2),
        stop_words='english',
        sublinear_tf=True
    )),
    ('clf', MultinomialNB(alpha=0.1))
])

svm_pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        max_features=10000,
        ngram_range=(1, 2),
        stop_words='english',
        sublinear_tf=True
    )),
    ('clf', LinearSVC(C=1.0, max_iter=10000))
])

# ─── Step 3: Train and Evaluate ───────────────────────────
results = {}

for name, pipeline in [('Naive Bayes', nb_pipeline), ('SVM', svm_pipeline)]:
    print(f"\n{'='*50}")
    print(f"  {name}")
    print(f"{'='*50}")
    
    # Training
    t0 = time.time()
    pipeline.fit(data_train.data, data_train.target)
    train_time = time.time() - t0
    
    # Prediction
    t0 = time.time()
    y_pred = pipeline.predict(data_test.data)
    pred_time = time.time() - t0
    
    # Metrics
    accuracy = (y_pred == data_test.target).mean()
    precision, recall, f1, _ = precision_recall_fscore_support(
        data_test.target, y_pred, average='binary'
    )
    
    results[name] = {
        'accuracy': accuracy,
        'precision': precision,
        'recall': recall,
        'f1': f1,
        'train_time': train_time,
        'pred_time': pred_time
    }
    
    print(f"  Accuracy:   {accuracy:.4f}")
    print(f"  Precision:  {precision:.4f}")
    print(f"  Recall:     {recall:.4f}")
    print(f"  F1-Score:   {f1:.4f}")
    print(f"  Train Time: {train_time:.3f}s")
    print(f"  Pred Time:  {pred_time:.3f}s")

# ─── Step 4: Compare Results ──────────────────────────────
print(f"\n{'='*50}")
print("  HEAD-TO-HEAD COMPARISON")
print(f"{'='*50}")
print(f"{'Metric':<15} {'Naive Bayes':<15} {'SVM':<15}")
print("-" * 45)
for metric in ['accuracy', 'precision', 'recall', 'f1']:
    nb_val = results['Naive Bayes'][metric]
    svm_val = results['SVM'][metric]
    winner = "← NB" if nb_val > svm_val else "← SVM"
    print(f"{metric:<15} {nb_val:<15.4f} {svm_val:<15.4f} {winner}")

# ─── Step 5: Test Custom Emails ───────────────────────────
print(f"\n{'='*50}")
print("  CUSTOM EMAIL CLASSIFICATION")
print(f"{'='*50}")

test_emails = [
    "The hockey game last night was incredible, great goals scored",
    "Patient showed symptoms of inflammation, prescribed antibiotics",
    "Team won the championship after overtime victory",
    "New research on cardiovascular disease treatment published",
]

for email in test_emails:
    nb_pred = data_train.target_names[nb_pipeline.predict([email])[0]]
    svm_pred = data_train.target_names[svm_pipeline.predict([email])[0]]
    print(f"\n  Email: \"{email[:60]}...\"")
    print(f"  NB:  {nb_pred}")
    print(f"  SVM: {svm_pred}")

"""
MINI PROJECT 2: Handwritten Digit Recognizer
=============================================
Compare KNN, SVM, and NB on digit classification
with dimensionality reduction via PCA.
"""

import numpy as np
from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import classification_report, confusion_matrix
import time

# ─── Step 1: Load and Explore Data ────────────────────────
digits = load_digits()
X, y = digits.data, digits.target

print("HANDWRITTEN DIGIT RECOGNIZER")
print("=" * 50)
print(f"Dataset shape: {X.shape}")
print(f"Feature range: [{X.min()}, {X.max()}]")
print(f"Classes: {np.unique(y)}")
print(f"Samples per class: {np.bincount(y)}")

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# ─── Step 2: Build Pipelines ──────────────────────────────
pipelines = {
    'KNN (K=3)': Pipeline([
        ('scaler', StandardScaler()),
        ('pca', PCA(n_components=0.95)),  # Keep 95% variance
        ('clf', KNeighborsClassifier(n_neighbors=3, weights='distance'))
    ]),
    'KNN (K=7)': Pipeline([
        ('scaler', StandardScaler()),
        ('pca', PCA(n_components=0.95)),
        ('clf', KNeighborsClassifier(n_neighbors=7, weights='distance'))
    ]),
    'SVM (RBF)': Pipeline([
        ('scaler', StandardScaler()),
        ('pca', PCA(n_components=0.95)),
        ('clf', SVC(kernel='rbf', C=10, gamma='scale'))
    ]),
    'SVM (Linear)': Pipeline([
        ('scaler', StandardScaler()),
        ('pca', PCA(n_components=0.95)),
        ('clf', SVC(kernel='linear', C=1.0))
    ]),
    'Gaussian NB': Pipeline([
        ('scaler', StandardScaler()),
        ('pca', PCA(n_components=0.95)),
        ('clf', GaussianNB())
    ]),
}

# ─── Step 3: Train and Evaluate ───────────────────────────
print(f"\n{'Model':<18} {'Train Acc':<12} {'Test Acc':<12} "
      f"{'Time (s)':<12} {'PCA Dims'}")
print("-" * 65)

best_model = None
best_acc = 0

for name, pipeline in pipelines.items():
    t0 = time.time()
    pipeline.fit(X_train, y_train)
    elapsed = time.time() - t0
    
    train_acc = pipeline.score(X_train, y_train)
    test_acc = pipeline.score(X_test, y_test)
    pca_dims = pipeline.named_steps['pca'].n_components_
    
    print(f"{name:<18} {train_acc:<12.4f} {test_acc:<12.4f} "
          f"{elapsed:<12.4f} {pca_dims}")
    
    if test_acc > best_acc:
        best_acc = test_acc
        best_model = name

print(f"\n🏆 Best model: {best_model} with {best_acc:.4f} accuracy")

# ─── Step 4: Detailed Analysis of Best Model ──────────────
print(f"\n{'='*50}")
print(f"  Detailed Report: {best_model}")
print(f"{'='*50}")

best_pipeline = pipelines[best_model]
y_pred = best_pipeline.predict(X_test)
print(classification_report(y_test, y_pred))

# Error analysis
errors = np.where(y_pred != y_test)[0]
print(f"\nTotal errors: {len(errors)} / {len(y_test)} "
      f"({100*len(errors)/len(y_test):.1f}%)")

if len(errors) > 0:
    print("\nSample errors:")
    for idx in errors[:10]:
        print(f"  Image {idx}: True={y_test[idx]}, "
              f"Predicted={y_pred[idx]}")

# ─── Step 5: Effect of PCA Components ─────────────────────
print(f"\n{'='*50}")
print("  PCA DIMENSIONALITY REDUCTION EFFECT")
print(f"{'='*50}")

for n_comp in [10, 20, 30, 40, 50, 64]:
    pipe = Pipeline([
        ('scaler', StandardScaler()),
        ('pca', PCA(n_components=n_comp)),
        ('clf', SVC(kernel='rbf', C=10, gamma='scale'))
    ])
    pipe.fit(X_train, y_train)
    acc = pipe.score(X_test, y_test)
    var_explained = pipe.named_steps['pca'].explained_variance_ratio_.sum()
    print(f"  Components={n_comp:3d} | "
          f"Variance={var_explained:.4f} | Accuracy={acc:.4f}")