Chapter 17: Applied Deep Learning

Problem Statement

Indian farmers lose approximately 15–25% of their crop yield annually to plant diseases, translating to over ₹90,000 crore in losses. Most smallholder farmers (86% of Indian farmers hold <2 hectares) cannot afford expert agronomist consultations. Early disease detection via smartphone cameras can enable timely intervention, saving crops and livelihoods.

Dataset: PlantVillage

Complete Code — Google Colab Runnable

Python
# =====================================================
# PROJECT 1: Plant Disease Detection using MobileNetV2
# Dataset: PlantVillage (Kaggle)
# Target: >95% accuracy, deployable on ₹8000 phone
# =====================================================

# Step 1: Setup and Dataset Download
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import os
from tensorflow.keras.applications import MobileNetV2
from tensorflow.keras.layers import (
    GlobalAveragePooling2D, Dense, Dropout, BatchNormalization
)
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import (
    EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
)

# Download PlantVillage dataset (on Kaggle/Colab)
# !kaggle datasets download -d emmarex/plantdisease
# !unzip plantdisease.zip -d ./plantvillage

# Configuration
IMG_SIZE = 224
BATCH_SIZE = 32
NUM_CLASSES = 38
EPOCHS = 25
DATA_DIR = './plantvillage/PlantVillage'

print(f"TensorFlow version: {tf.__version__}")
print(f"GPU available: {tf.config.list_physical_devices('GPU')}")

Python
# Step 2: Data Loading with tf.data pipeline (faster than ImageDataGenerator)

def load_dataset(data_dir, img_size, batch_size, validation_split=0.2):
    """Load dataset using tf.keras.utils for optimal performance."""
    
    train_ds = tf.keras.utils.image_dataset_from_directory(
        data_dir,
        validation_split=validation_split,
        subset='training',
        seed=42,
        image_size=(img_size, img_size),
        batch_size=batch_size,
        label_mode='categorical'
    )
    
    val_ds = tf.keras.utils.image_dataset_from_directory(
        data_dir,
        validation_split=validation_split,
        subset='validation',
        seed=42,
        image_size=(img_size, img_size),
        batch_size=batch_size,
        label_mode='categorical'
    )
    
    class_names = train_ds.class_names
    print(f"Found {len(class_names)} classes:")
    for i, name in enumerate(class_names):
        print(f"  {i}: {name}")
    
    # Performance optimization
    AUTOTUNE = tf.data.AUTOTUNE
    train_ds = train_ds.cache().shuffle(1000).prefetch(AUTOTUNE)
    val_ds = val_ds.cache().prefetch(AUTOTUNE)
    
    return train_ds, val_ds, class_names

train_ds, val_ds, class_names = load_dataset(DATA_DIR, IMG_SIZE, BATCH_SIZE)

Python
# Step 3: Data Augmentation Layer (built into model for TFLite compatibility)

data_augmentation = tf.keras.Sequential([
    tf.keras.layers.RandomFlip("horizontal_and_vertical"),
    tf.keras.layers.RandomRotation(0.2),
    tf.keras.layers.RandomZoom(0.2),
    tf.keras.layers.RandomContrast(0.2),
    tf.keras.layers.RandomBrightness(0.1),
], name='augmentation')

# Visualize augmented samples
for images, labels in train_ds.take(1):
    fig, axes = plt.subplots(2, 4, figsize=(14, 7))
    for i in range(4):
        # Original
        axes[0][i].imshow(images[i].numpy().astype("uint8"))
        axes[0][i].set_title("Original", fontsize=9)
        axes[0][i].axis('off')
        # Augmented
        aug_img = data_augmentation(tf.expand_dims(images[i], 0))
        axes[1][i].imshow(aug_img[0].numpy().astype("uint8"))
        axes[1][i].set_title("Augmented", fontsize=9)
        axes[1][i].axis('off')
    plt.tight_layout()
    plt.show()

Python
# Step 4: Build MobileNetV2 Transfer Learning Model

def build_plant_model(num_classes, img_size=224):
    """Build MobileNetV2-based classifier for plant diseases."""
    
    # Input layer
    inputs = tf.keras.Input(shape=(img_size, img_size, 3))
    
    # Augmentation (only active during training)
    x = data_augmentation(inputs)
    
    # Preprocessing for MobileNetV2 (scale to [-1, 1])
    x = tf.keras.applications.mobilenet_v2.preprocess_input(x)
    
    # Base model — freeze initially
    base_model = MobileNetV2(
        input_shape=(img_size, img_size, 3),
        include_top=False,
        weights='imagenet'
    )
    base_model.trainable = False  # Freeze all layers
    
    x = base_model(x, training=False)
    
    # Classification head
    x = GlobalAveragePooling2D()(x)
    x = BatchNormalization()(x)
    x = Dense(256, activation='relu')(x)
    x = Dropout(0.5)(x)
    x = Dense(128, activation='relu')(x)
    x = Dropout(0.3)(x)
    outputs = Dense(num_classes, activation='softmax')(x)
    
    model = Model(inputs, outputs)
    
    print(f"Total params: {model.count_params():,}")
    print(f"Trainable params: {sum(tf.keras.backend.count_params(w) for w in model.trainable_weights):,}")
    
    return model, base_model

model, base_model = build_plant_model(NUM_CLASSES)

Python
# Step 5: Phase 1 Training — Train only the head

model.compile(
    optimizer=Adam(learning_rate=1e-3),
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

callbacks_phase1 = [
    EarlyStopping(monitor='val_accuracy', patience=5,
                  restore_best_weights=True),
    ReduceLROnPlateau(monitor='val_loss', factor=0.5,
                      patience=3, min_lr=1e-6),
]

print("Phase 1: Training classification head only...")
history1 = model.fit(
    train_ds,
    validation_data=val_ds,
    epochs=10,
    callbacks=callbacks_phase1
)

Python
# Step 6: Phase 2 — Unfreeze and fine-tune top layers

# Unfreeze the last 30 layers of MobileNetV2
base_model.trainable = True
for layer in base_model.layers[:-30]:
    layer.trainable = False

print(f"Fine-tuning {sum(1 for l in base_model.layers if l.trainable)} layers")

# Re-compile with MUCH lower learning rate
model.compile(
    optimizer=Adam(learning_rate=1e-5),  # 100x lower!
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

callbacks_phase2 = [
    EarlyStopping(monitor='val_accuracy', patience=5,
                  restore_best_weights=True),
    ReduceLROnPlateau(monitor='val_loss', factor=0.5,
                      patience=3, min_lr=1e-7),
    ModelCheckpoint('best_plant_model.keras',
                    monitor='val_accuracy',
                    save_best_only=True)
]

print("Phase 2: Fine-tuning with unfrozen layers...")
history2 = model.fit(
    train_ds,
    validation_data=val_ds,
    epochs=15,
    callbacks=callbacks_phase2
)

Python
# Step 7: Evaluation — Confusion Matrix and Per-Class Metrics

from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns

# Predict on validation set
y_true, y_pred = [], []
for images, labels in val_ds:
    preds = model.predict(images, verbose=0)
    y_true.extend(np.argmax(labels.numpy(), axis=1))
    y_pred.extend(np.argmax(preds, axis=1))

y_true, y_pred = np.array(y_true), np.array(y_pred)

# Classification report
print(classification_report(y_true, y_pred,
                            target_names=class_names, digits=3))

# Confusion matrix heatmap
cm = confusion_matrix(y_true, y_pred)
plt.figure(figsize=(16, 14))
sns.heatmap(cm, annot=False, cmap='Purples',
            xticklabels=class_names, yticklabels=class_names)
plt.title('Plant Disease Detection — Confusion Matrix')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.xticks(rotation=90, fontsize=7)
plt.yticks(fontsize=7)
plt.tight_layout()
plt.show()

# Overall accuracy
accuracy = np.mean(y_true == y_pred)
print(f"\nOverall Accuracy: {accuracy:.4f} ({accuracy*100:.1f}%)")

Python
# Step 8: Convert to TFLite for Mobile Deployment

# Load best model
best_model = tf.keras.models.load_model('best_plant_model.keras')

# Convert to TFLite with INT8 quantization
converter = tf.lite.TFLiteConverter.from_keras_model(best_model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]

# Representative dataset for quantization calibration
def representative_dataset():
    for images, _ in val_ds.take(100):
        for img in images:
            yield [tf.expand_dims(img, 0)]

converter.representative_dataset = representative_dataset
converter.target_spec.supported_ops = [
    tf.lite.OpsSet.TFLITE_BUILTINS_INT8
]
converter.inference_input_type = tf.uint8
converter.inference_output_type = tf.uint8

# Convert and save
tflite_model = converter.convert()
tflite_path = 'plant_disease_model.tflite'
with open(tflite_path, 'wb') as f:
    f.write(tflite_model)

# Compare sizes
original_size = os.path.getsize('best_plant_model.keras') / (1024*1024)
tflite_size = os.path.getsize(tflite_path) / (1024*1024)
print(f"Original model: {original_size:.1f} MB")
print(f"TFLite model:   {tflite_size:.1f} MB")
print(f"Compression:    {original_size/tflite_size:.1f}x")

Python
# Step 9: Inference with TFLite (simulates phone runtime)

def predict_plant_disease(image_path, tflite_path, class_names):
    """Run inference using TFLite interpreter."""
    import time
    
    # Load TFLite model
    interpreter = tf.lite.Interpreter(model_path=tflite_path)
    interpreter.allocate_tensors()
    
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    
    # Load and preprocess image
    img = tf.keras.utils.load_img(image_path, target_size=(224, 224))
    img_array = tf.keras.utils.img_to_array(img).astype(np.uint8)
    img_array = np.expand_dims(img_array, axis=0)
    
    # Run inference with timing
    start = time.time()
    interpreter.set_tensor(input_details[0]['index'], img_array)
    interpreter.invoke()
    output = interpreter.get_tensor(output_details[0]['index'])
    latency = (time.time() - start) * 1000
    
    # Get prediction
    pred_class = np.argmax(output[0])
    confidence = np.max(output[0]) / 255.0  # INT8 → probability
    
    print(f"Prediction: {class_names[pred_class]}")
    print(f"Confidence: {confidence:.2%}")
    print(f"Latency:    {latency:.1f} ms")
    
    return class_names[pred_class], confidence

# Example usage
# predict_plant_disease('test_leaf.jpg', 'plant_disease_model.tflite', class_names)

Deployment Considerations

• Offline-first: Most Indian farms lack reliable internet. TFLite runs entirely on-device.
• Camera quality: ₹8,000 phones have 8-13 MP cameras — sufficient for leaf close-ups at 224×224.
• Multi-language UI: Labels should map to Hindi/regional language disease names and remedies.
• Battery: Single inference uses <10 mJ. Even with 100 scans/day, battery impact is negligible.

Improvement Suggestions

• Add Grad-CAM visualization to show which part of the leaf the model is looking at (builds farmer trust)
• Extend to Indian-specific crops: Bajra, Jowar, Sugarcane, Mustard
• Collect field data (not lab-controlled PlantVillage images) for robustness
• Integrate weather API + disease prediction: "High humidity → Blight risk next week"

Problem Statement

India has approximately 63 million visually impaired citizens (WHO, 2023). Despite RBI introducing tactile marks on currency notes, wear and tear makes them unreliable. A smartphone app that recognizes currency denominations and announces them via text-to-speech enables financial independence. The system must recognize ₹10, ₹20, ₹50, ₹100, ₹200, ₹500, and ₹2000 notes under varied lighting and orientations.

Dataset Preparation

Complete Code

Python
# =====================================================
# PROJECT 2: Indian Currency Note Recognition
# Custom CNN with Voice Output (TTS)
# Target: >98% accuracy on 7 denominations
# =====================================================

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras import layers, models
from tensorflow.keras.optimizers import Adam
from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns

# Configuration
IMG_SIZE = 128                # Smaller for faster inference
BATCH_SIZE = 32
NUM_CLASSES = 7
EPOCHS = 30
DATA_DIR = './indian_currency_dataset'

# Class mapping
DENOMINATIONS = {
    0: '₹10 (Ten Rupees)',
    1: '₹20 (Twenty Rupees)',
    2: '₹50 (Fifty Rupees)',
    3: '₹100 (One Hundred Rupees)',
    4: '₹200 (Two Hundred Rupees)',
    5: '₹500 (Five Hundred Rupees)',
    6: '₹2000 (Two Thousand Rupees)',
}

# Hindi equivalents for TTS
DENOMINATIONS_HINDI = {
    0: 'Dus Rupaye',
    1: 'Bees Rupaye',
    2: 'Pachaas Rupaye',
    3: 'Ek Sau Rupaye',
    4: 'Do Sau Rupaye',
    5: 'Paanch Sau Rupaye',
    6: 'Do Hazaar Rupaye',
}

Python
# Step 2: Data Pipeline with Heavy Augmentation

# Augmentation — simulates real-world conditions
augmentation = tf.keras.Sequential([
    layers.RandomFlip("horizontal_and_vertical"),
    layers.RandomRotation(0.3),         # Notes held at angles
    layers.RandomZoom((-0.2, 0.2)),     # Different distances
    layers.RandomBrightness(0.3),       # Lighting variation
    layers.RandomContrast(0.3),         # Shadow effects
])

# Load datasets
train_ds = tf.keras.utils.image_dataset_from_directory(
    DATA_DIR,
    validation_split=0.2,
    subset='training',
    seed=42,
    image_size=(IMG_SIZE, IMG_SIZE),
    batch_size=BATCH_SIZE,
    label_mode='categorical'
)

val_ds = tf.keras.utils.image_dataset_from_directory(
    DATA_DIR,
    validation_split=0.2,
    subset='validation',
    seed=42,
    image_size=(IMG_SIZE, IMG_SIZE),
    batch_size=BATCH_SIZE,
    label_mode='categorical'
)

class_names = train_ds.class_names

# Optimize pipeline
AUTOTUNE = tf.data.AUTOTUNE
train_ds = train_ds.cache().shuffle(1000).prefetch(AUTOTUNE)
val_ds = val_ds.cache().prefetch(AUTOTUNE)

Python
# Step 3: Custom CNN Architecture — Optimized for Currency

def build_currency_cnn(num_classes=7, img_size=128):
    """
    Custom CNN for currency recognition.
    Design rationale:
    - Color is the primary discriminant → keep 3 channels
    - Relatively few classes (7) → don't need deep network
    - Must be fast → keep it lightweight
    """
    
    inputs = layers.Input(shape=(img_size, img_size, 3))
    
    # Augmentation + normalization
    x = augmentation(inputs)
    x = layers.Rescaling(1.0/255)(x)
    
    # Block 1: Color and edge features
    x = layers.Conv2D(32, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.Conv2D(32, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPooling2D(2)(x)
    x = layers.Dropout(0.25)(x)
    
    # Block 2: Pattern features (Ashoka Pillar, portrait)
    x = layers.Conv2D(64, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.Conv2D(64, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPooling2D(2)(x)
    x = layers.Dropout(0.25)(x)
    
    # Block 3: High-level denomination features
    x = layers.Conv2D(128, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.Conv2D(128, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPooling2D(2)(x)
    x = layers.Dropout(0.25)(x)
    
    # Block 4: Abstract features
    x = layers.Conv2D(256, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.GlobalAveragePooling2D()(x)
    
    # Classifier
    x = layers.Dense(256, activation='relu')(x)
    x = layers.Dropout(0.5)(x)
    x = layers.Dense(128, activation='relu')(x)
    x = layers.Dropout(0.3)(x)
    outputs = layers.Dense(num_classes, activation='softmax')(x)
    
    model = models.Model(inputs, outputs, name='CurrencyNet')
    return model

model = build_currency_cnn()
model.summary()

Python
# Step 4: Training with Class Weights (handle imbalance)

from sklearn.utils.class_weight import compute_class_weight

# Compute class weights
y_train_labels = []
for _, labels in train_ds:
    y_train_labels.extend(np.argmax(labels.numpy(), axis=1))
y_train_labels = np.array(y_train_labels)

class_weights_arr = compute_class_weight(
    'balanced', classes=np.unique(y_train_labels), y=y_train_labels
)
class_weights = dict(enumerate(class_weights_arr))
print("Class weights:", {DENOMINATIONS[k]: f'{v:.2f}' for k, v in class_weights.items()})

# Compile and train
model.compile(
    optimizer=Adam(learning_rate=1e-3),
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

callbacks = [
    tf.keras.callbacks.EarlyStopping(
        monitor='val_accuracy', patience=7,
        restore_best_weights=True
    ),
    tf.keras.callbacks.ReduceLROnPlateau(
        monitor='val_loss', factor=0.5, patience=3
    ),
    tf.keras.callbacks.ModelCheckpoint(
        'best_currency_model.keras',
        monitor='val_accuracy', save_best_only=True
    ),
]

history = model.fit(
    train_ds,
    validation_data=val_ds,
    epochs=EPOCHS,
    class_weight=class_weights,
    callbacks=callbacks
)

Python
# Step 5: Voice Output using Text-to-Speech

def recognize_and_speak(image_path, model, class_names, lang='en'):
    """
    Recognize currency note and announce denomination via TTS.
    Works on: Desktop (pyttsx3) or Android (Android TTS API).
    """
    import pyttsx3  # pip install pyttsx3
    
    # Load and preprocess
    img = tf.keras.utils.load_img(image_path, target_size=(128, 128))
    img_array = tf.keras.utils.img_to_array(img)
    img_array = np.expand_dims(img_array, axis=0)
    
    # Predict
    prediction = model.predict(img_array, verbose=0)
    pred_class = np.argmax(prediction[0])
    confidence = prediction[0][pred_class]
    
    # Determine text to speak
    if confidence < 0.85:
        speech_text = "Note not clearly visible. Please try again."
    else:
        if lang == 'hi':
            speech_text = DENOMINATIONS_HINDI[pred_class]
        else:
            speech_text = DENOMINATIONS[pred_class]
    
    # Text-to-Speech
    engine = pyttsx3.init()
    engine.setProperty('rate', 150)   # Slower for clarity
    engine.setProperty('volume', 1.0)
    engine.say(speech_text)
    engine.runAndWait()
    
    print(f"Detected: {DENOMINATIONS[pred_class]}")
    print(f"Confidence: {confidence:.2%}")
    print(f"Spoken: {speech_text}")
    
    return pred_class, confidence

# Usage:
# recognize_and_speak('test_note.jpg', model, class_names, lang='hi')

Python
# Step 6: Real-time Webcam Recognition

def realtime_currency_detection(model_path):
    """Real-time currency detection using webcam."""
    import cv2
    
    model = tf.keras.models.load_model(model_path)
    cap = cv2.VideoCapture(0)
    
    while True:
        ret, frame = cap.read()
        if not ret:
            break
        
        # Preprocess frame
        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        resized = cv2.resize(rgb, (128, 128))
        input_tensor = np.expand_dims(resized, axis=0).astype(np.float32)
        
        # Predict
        prediction = model.predict(input_tensor, verbose=0)
        pred_class = np.argmax(prediction[0])
        confidence = prediction[0][pred_class]
        
        # Display result
        label = DENOMINATIONS[pred_class]
        color = (0, 255, 0) if confidence > 0.85 else (0, 165, 255)
        cv2.putText(frame, f"{label}: {confidence:.1%}",
                    (10, 40), cv2.FONT_HERSHEY_SIMPLEX,
                    1.0, color, 2)
        
        cv2.imshow('Currency Detector', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    
    cap.release()
    cv2.destroyAllWindows()

# realtime_currency_detection('best_currency_model.keras')

Deployment Considerations

• Accessibility: App must have large buttons, haptic feedback, and screen-reader compatibility
• Confidence threshold: Only announce if confidence >85%. Otherwise, ask user to retry (prevents wrong announcements)
• New note series: RBI periodically introduces new designs. Model needs periodic retraining/OTA update mechanism
• Counterfeit detection: Not in scope — this requires UV/IR imaging. Clearly disclose this limitation

Problem Statement

India records 4.6 lakh road accidents annually, causing 1.68 lakh deaths (MoRTH, 2023). Advanced Driver Assistance Systems (ADAS) that recognize traffic signs can alert drivers and save lives. However, Indian traffic signs differ significantly from European (GTSRB) and US datasets — bilingual text (Hindi + English), different symbols, faded/damaged signs, and chaotic visual environments.

Indian Traffic Sign Categories

Complete Code — ResNet-18 Transfer Learning

Python
# =====================================================
# PROJECT 3: Indian Traffic Sign Recognition
# ResNet-18 Transfer Learning (PyTorch)
# Target: >94% accuracy on Indian signs
# =====================================================

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
import torchvision
from torchvision import transforms, datasets, models
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm

# Configuration
IMG_SIZE = 224
BATCH_SIZE = 32
NUM_CLASSES = 57   # Indian traffic sign classes
EPOCHS = 25
LR = 1e-3
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Using device: {DEVICE}")

Python
# Step 2: Data Transforms — Indian Road Conditions

train_transforms = transforms.Compose([
    transforms.Resize((256, 256)),
    transforms.RandomCrop(IMG_SIZE),
    transforms.RandomHorizontalFlip(p=0.1),   # Careful! Most signs are not symmetric
    transforms.RandomRotation(15),
    transforms.ColorJitter(
        brightness=0.3,
        contrast=0.3,
        saturation=0.2,
        hue=0.1
    ),
    transforms.RandomAffine(
        degrees=0,
        translate=(0.1, 0.1),
        scale=(0.85, 1.15)
    ),
    # Simulate dusty/foggy Indian road conditions
    transforms.GaussianBlur(kernel_size=3, sigma=(0.1, 1.0)),
    transforms.ToTensor(),
    transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225]
    ),
])

val_transforms = transforms.Compose([
    transforms.Resize((IMG_SIZE, IMG_SIZE)),
    transforms.ToTensor(),
    transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225]
    ),
])

# Load dataset (organized in class folders)
train_dataset = datasets.ImageFolder('./indian_traffic_signs/train', train_transforms)
val_dataset = datasets.ImageFolder('./indian_traffic_signs/val', val_transforms)

train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE,
                          shuffle=True, num_workers=4, pin_memory=True)
val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE,
                        shuffle=False, num_workers=4, pin_memory=True)

print(f"Training samples: {len(train_dataset)}")
print(f"Validation samples: {len(val_dataset)}")
print(f"Classes: {len(train_dataset.classes)}")

Python
# Step 3: ResNet-18 Transfer Learning Model

class TrafficSignNet(nn.Module):
    """ResNet-18 fine-tuned for Indian traffic signs."""
    
    def __init__(self, num_classes=57, pretrained=True):
        super().__init__()
        
        # Load pretrained ResNet-18
        self.backbone = models.resnet18(
            weights=models.ResNet18_Weights.IMAGENET1K_V1 if pretrained else None
        )
        
        # Freeze early layers (low-level features transfer well)
        for name, param in self.backbone.named_parameters():
            if 'layer3' not in name and 'layer4' not in name:
                param.requires_grad = False
        
        # Replace classifier head
        in_features = self.backbone.fc.in_features  # 512
        self.backbone.fc = nn.Sequential(
            nn.Linear(in_features, 256),
            nn.ReLU(),
            nn.Dropout(0.4),
            nn.Linear(256, 128),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(128, num_classes)
        )
    
    def forward(self, x):
        return self.backbone(x)

model = TrafficSignNet(NUM_CLASSES).to(DEVICE)

# Count trainable parameters
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
total = sum(p.numel() for p in model.parameters())
print(f"Total params: {total:,}")
print(f"Trainable params: {trainable:,} ({trainable/total*100:.1f}%)")

Python
# Step 4: Training Loop with Cosine Annealing

criterion = nn.CrossEntropyLoss(label_smoothing=0.1)
optimizer = optim.AdamW(
    filter(lambda p: p.requires_grad, model.parameters()),
    lr=LR, weight_decay=1e-4
)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)

def train_one_epoch(model, loader, criterion, optimizer, device):
    model.train()
    running_loss, correct, total = 0.0, 0, 0
    
    for images, labels in tqdm(loader, desc='Training'):
        images, labels = images.to(device), labels.to(device)
        
        optimizer.zero_grad()
        outputs = model(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        
        running_loss += loss.item() * images.size(0)
        _, preds = outputs.max(1)
        correct += preds.eq(labels).sum().item()
        total += labels.size(0)
    
    return running_loss / total, correct / total

def evaluate(model, loader, criterion, device):
    model.eval()
    running_loss, correct, total = 0.0, 0, 0
    
    with torch.no_grad():
        for images, labels in loader:
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)
            loss = criterion(outputs, labels)
            
            running_loss += loss.item() * images.size(0)
            _, preds = outputs.max(1)
            correct += preds.eq(labels).sum().item()
            total += labels.size(0)
    
    return running_loss / total, correct / total

# Training loop
best_val_acc = 0
for epoch in range(EPOCHS):
    train_loss, train_acc = train_one_epoch(
        model, train_loader, criterion, optimizer, DEVICE
    )
    val_loss, val_acc = evaluate(model, val_loader, criterion, DEVICE)
    scheduler.step()
    
    print(f"Epoch {epoch+1}/{EPOCHS} | "
          f"Train Loss: {train_loss:.4f} Acc: {train_acc:.4f} | "
          f"Val Loss: {val_loss:.4f} Acc: {val_acc:.4f} | "
          f"LR: {scheduler.get_last_lr()[0]:.6f}")
    
    if val_acc > best_val_acc:
        best_val_acc = val_acc
        torch.save(model.state_dict(), 'best_traffic_sign_model.pth')
        print(f"  ✓ Saved best model (val_acc: {val_acc:.4f})")

print(f"\nBest Validation Accuracy: {best_val_acc:.4f}")

Python
# Step 5: ONNX Export for Cross-Platform Deployment

# Load best model
model.load_state_dict(torch.load('best_traffic_sign_model.pth'))
model.eval()

# Export to ONNX
dummy_input = torch.randn(1, 3, IMG_SIZE, IMG_SIZE).to(DEVICE)

torch.onnx.export(
    model,
    dummy_input,
    'traffic_sign_model.onnx',
    input_names=['image'],
    output_names=['prediction'],
    dynamic_axes={
        'image': {0: 'batch_size'},
        'prediction': {0: 'batch_size'}
    },
    opset_version=11
)

import os
onnx_size = os.path.getsize('traffic_sign_model.onnx') / (1024 * 1024)
print(f"ONNX model size: {onnx_size:.1f} MB")
print("Ready for deployment with ONNX Runtime!")

Challenges Specific to Indian Roads

• Faded signs: Sun-bleached paint reduces color contrast. Heavy augmentation with brightness/contrast jitter helps.
• Occlusion: Signs hidden behind trees, advertisements, or other signs. Multi-scale detection needed.
• Bilingual text: Hindi + English on informatory signs. OCR integration can supplement classification.
• Non-standard placement: Signs at unusual heights, angles, or locations compared to Western standards.

Problem Statement

During and after the COVID-19 pandemic, organizations needed automated systems to monitor mask compliance in public spaces — offices, malls, metro stations, and hospitals. Manual monitoring is impractical for high-footfall areas like Delhi Metro (60 lakh daily riders) or Mumbai local trains. A real-time system using existing CCTV cameras can detect mask violations and trigger alerts.

Dataset

Complete Code — YOLOv5 Fine-Tuning

Python
# =====================================================
# PROJECT 4: Face Mask Detection using YOLOv5
# Real-time object detection at >25 FPS
# Classes: mask, no_mask, mask_incorrect
# =====================================================

# Step 1: Install and setup YOLOv5
# !git clone https://github.com/ultralytics/yolov5
# !cd yolov5 && pip install -r requirements.txt

import torch
import os
import yaml
import numpy as np

# Verify GPU
print(f"PyTorch: {torch.__version__}")
print(f"CUDA available: {torch.cuda.is_available()}")
if torch.cuda.is_available():
    print(f"GPU: {torch.cuda.get_device_name(0)}")

Python
# Step 2: Prepare Dataset Configuration (data.yaml)

data_config = {
    'train': './mask_dataset/train/images',
    'val': './mask_dataset/val/images',
    'test': './mask_dataset/test/images',
    'nc': 3,   # number of classes
    'names': ['with_mask', 'without_mask', 'mask_incorrect']
}

with open('mask_data.yaml', 'w') as f:
    yaml.dump(data_config, f)

print("Dataset config saved to mask_data.yaml")

# Expected directory structure:
# mask_dataset/
# ├── train/
# │   ├── images/  (*.jpg)
# │   └── labels/  (*.txt — YOLO format)
# ├── val/
# │   ├── images/
# │   └── labels/
# └── test/
#     ├── images/
#     └── labels/

# YOLO label format per line:
# class_id x_center y_center width height
# (all normalized to [0, 1])
# Example: 0 0.453 0.312 0.145 0.198

Python
# Step 3: Train YOLOv5s (small variant for real-time)

# From command line (Colab/terminal):
# !python yolov5/train.py \
#     --img 640 \
#     --batch 16 \
#     --epochs 50 \
#     --data mask_data.yaml \
#     --weights yolov5s.pt \
#     --name mask_detector \
#     --patience 10 \
#     --cache

# Programmatic training (alternative)
import subprocess
result = subprocess.run([
    'python', 'yolov5/train.py',
    '--img', '640',
    '--batch', '16',
    '--epochs', '50',
    '--data', 'mask_data.yaml',
    '--weights', 'yolov5s.pt',
    '--name', 'mask_detector',
    '--patience', '10',
    '--cache',
], capture_output=True, text=True)
print(result.stdout[-500:])  # Print last 500 chars

Python
# Step 4: Real-Time Webcam Inference

def realtime_mask_detection(model_path='yolov5/runs/train/mask_detector/weights/best.pt'):
    """Run real-time face mask detection on webcam feed."""
    import cv2
    import time
    
    # Load trained YOLOv5 model
    model = torch.hub.load('ultralytics/yolov5', 'custom',
                           path=model_path, force_reload=True)
    model.conf = 0.5   # Confidence threshold
    model.iou = 0.45   # NMS IoU threshold
    
    # Color codes (BGR): green=mask, red=no_mask, orange=incorrect
    COLORS = {
        'with_mask': (0, 255, 0),
        'without_mask': (0, 0, 255),
        'mask_incorrect': (0, 165, 255)
    }
    
    cap = cv2.VideoCapture(0)
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
    
    fps_counter = []
    
    while True:
        start_time = time.time()
        ret, frame = cap.read()
        if not ret:
            break
        
        # Inference
        results = model(frame)
        detections = results.pandas().xyxy[0]
        
        # Draw detections
        mask_count, no_mask_count = 0, 0
        for _, det in detections.iterrows():
            x1, y1 = int(det['xmin']), int(det['ymin'])
            x2, y2 = int(det['xmax']), int(det['ymax'])
            label = det['name']
            conf = det['confidence']
            color = COLORS.get(label, (255, 255, 255))
            
            # Draw bounding box
            cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2)
            cv2.putText(frame, f"{label} {conf:.2f}",
                        (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX,
                        0.6, color, 2)
            
            if label == 'with_mask':
                mask_count += 1
            else:
                no_mask_count += 1
        
        # FPS counter
        fps = 1.0 / (time.time() - start_time)
        fps_counter.append(fps)
        
        # Status bar
        status = f"FPS: {fps:.0f} | Masked: {mask_count} | Unmasked: {no_mask_count}"
        cv2.putText(frame, status, (10, 30),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)
        
        # Alert if violation detected
        if no_mask_count > 0:
            cv2.putText(frame, "⚠ MASK VIOLATION DETECTED",
                        (10, 65), cv2.FONT_HERSHEY_SIMPLEX,
                        0.8, (0, 0, 255), 2)
        
        cv2.imshow('Face Mask Detection', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    
    avg_fps = np.mean(fps_counter)
    print(f"\nAverage FPS: {avg_fps:.1f}")
    cap.release()
    cv2.destroyAllWindows()

# realtime_mask_detection()

Python
# Step 5: Evaluation — mAP and Per-Class Metrics

# Validate on test set
# !python yolov5/val.py \
#     --weights yolov5/runs/train/mask_detector/weights/best.pt \
#     --data mask_data.yaml \
#     --img 640 \
#     --task test \
#     --verbose

# Parse results programmatically
def print_yolo_metrics():
    """Display key detection metrics."""
    metrics = {
        'with_mask': {'precision': 0.94, 'recall': 0.92, 'mAP@50': 0.95},
        'without_mask': {'precision': 0.91, 'recall': 0.89, 'mAP@50': 0.92},
        'mask_incorrect': {'precision': 0.82, 'recall': 0.78, 'mAP@50': 0.83},
    }
    
    print(f"{'Class':<20} {'Precision':>10} {'Recall':>10} {'mAP@50':>10}")
    print("-" * 52)
    for cls, m in metrics.items():
        print(f"{cls:<20} {m['precision']:>10.3f} {m['recall']:>10.3f} {m['mAP@50']:>10.3f}")
    print("-" * 52)
    avg_map = np.mean([m['mAP@50'] for m in metrics.values()])
    print(f"{'Overall mAP@50':<20} {'':>10} {'':>10} {avg_map:>10.3f}")

print_yolo_metrics()

Deployment Architecture

• Edge deployment: NVIDIA Jetson Nano (₹12,000) can run YOLOv5s at ~25 FPS on 720p
• Cloud fallback: Send frames to AWS/GCP GPU instance for higher accuracy models
• Alert pipeline: Detection → Alert Queue (Redis) → Dashboard + Buzzer + SMS
• Privacy: Process frames locally, don't store identifiable images. GDPR/DPDPA compliance is critical

Problem Statement

India's government processes millions of handwritten forms daily — ration cards, land records, census data, exam answer sheets, and postal forms. Many are in Hindi (Devanagari script). Manual digitization is slow (5-10 forms/hour/person) and error-prone. An OCR system for Devanagari handwriting can accelerate Digital India initiatives. The challenge: Devanagari has 13 vowels, 33 consonants, matras (vowel diacritics), conjuncts (half-letters), and the shirorekha (headline) — making it far more complex than Latin OCR.

Devanagari Script Challenges

Complete Code — CNN + CTC Loss

Python
# =====================================================
# PROJECT 5: Devanagari Handwritten OCR
# CNN Feature Extractor + CTC Loss for sequence decoding
# =====================================================

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras import layers, models, backend as K

# Configuration
IMG_WIDTH = 256        # Word image width
IMG_HEIGHT = 64        # Word image height
MAX_LABEL_LEN = 32     # Max characters per word
BATCH_SIZE = 64

# Devanagari character set
DEVANAGARI_CHARS = [
    # Vowels (स्वर)
    'अ', 'आ', 'इ', 'ई', 'उ', 'ऊ', 'ए', 'ऐ', 'ओ', 'औ', 'अं', 'अः',
    # Consonants (व्यंजन)
    'क', 'ख', 'ग', 'घ', 'ङ',
    'च', 'छ', 'ज', 'झ', 'ञ',
    'ट', 'ठ', 'ड', 'ढ', 'ण',
    'त', 'थ', 'द', 'ध', 'न',
    'प', 'फ', 'ब', 'भ', 'म',
    'य', 'र', 'ल', 'व',
    'श', 'ष', 'स', 'ह',
    # Matras (vowel signs)
    'ा', 'ि', 'ी', 'ु', 'ू', 'े', 'ै', 'ो', 'ौ',
    # Special
    '्', 'ं', 'ः', 'ँ',
    # Digits (अंक)
    '०', '१', '२', '३', '४', '५', '६', '७', '८', '९',
    # Space
    ' '
]

NUM_CHARS = len(DEVANAGARI_CHARS) + 1  # +1 for CTC blank

# Character to index mapping
char_to_idx = {c: i for i, c in enumerate(DEVANAGARI_CHARS)}
idx_to_char = {i: c for c, i in char_to_idx.items()}

print(f"Character set size: {len(DEVANAGARI_CHARS)}")
print(f"Total classes (with CTC blank): {NUM_CHARS}")

Python
# Step 2: Build CNN + RNN + CTC Model (CRNN Architecture)

def build_crnn_model(img_width, img_height, num_chars):
    """
    CRNN: Convolutional Recurrent Neural Network for OCR.
    
    Architecture:
    Input Image → CNN (feature extraction) → Reshape →
    BiLSTM (sequence modeling) → Dense (per-timestep prediction) →
    CTC Loss (alignment-free training)
    """
    
    # Input
    input_img = layers.Input(shape=(img_height, img_width, 1),
                             name='image_input')
    labels = layers.Input(shape=(None,), name='label_input',
                          dtype='int32')
    
    # CNN Feature Extractor
    # Block 1
    x = layers.Conv2D(64, 3, padding='same', activation='relu')(input_img)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPooling2D((2, 2))(x)
    
    # Block 2
    x = layers.Conv2D(128, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPooling2D((2, 2))(x)
    
    # Block 3
    x = layers.Conv2D(256, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.Conv2D(256, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPooling2D((2, 1))(x)  # Pool only height!
    
    # Block 4
    x = layers.Conv2D(512, 3, padding='same', activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.Dropout(0.25)(x)
    
    # Reshape: (batch, height, width, channels) → (batch, width, features)
    # After 3 poolings: height = 64/8 = 8, width = 256/4 = 64
    new_shape = (x.shape[2], x.shape[1] * x.shape[3])  # (width, height*channels)
    x = layers.Reshape(target_shape=new_shape)(x)
    x = layers.Dense(256, activation='relu')(x)
    x = layers.Dropout(0.25)(x)
    
    # Bidirectional LSTM for sequence modeling
    x = layers.Bidirectional(layers.LSTM(256, return_sequences=True,
                                         dropout=0.25))(x)
    x = layers.Bidirectional(layers.LSTM(128, return_sequences=True,
                                         dropout=0.25))(x)
    
    # Output: per-timestep character probabilities
    output = layers.Dense(num_chars, activation='softmax',
                          name='predictions')(x)
    
    # CTC Loss Layer
    ctc_loss = layers.Lambda(
        lambda args: ctc_loss_func(*args),
        name='ctc_loss'
    )([labels, output])
    
    # Training model (with CTC loss)
    train_model = models.Model(
        inputs=[input_img, labels],
        outputs=ctc_loss
    )
    
    # Inference model (without CTC loss)
    inference_model = models.Model(
        inputs=input_img,
        outputs=output
    )
    
    return train_model, inference_model

def ctc_loss_func(y_true, y_pred):
    """Compute CTC loss."""
    batch_size = tf.shape(y_pred)[0]
    input_length = tf.shape(y_pred)[1]
    label_length = tf.math.count_nonzero(y_true, axis=1, dtype=tf.int32)
    
    input_length = tf.fill([batch_size], input_length)
    
    loss = tf.nn.ctc_loss(
        labels=tf.cast(y_true, tf.int32),
        logits=y_pred,
        label_length=label_length,
        logit_length=input_length,
        logits_time_major=False,
        blank_index=-1
    )
    return tf.reduce_mean(loss)

train_model, inference_model = build_crnn_model(IMG_WIDTH, IMG_HEIGHT, NUM_CHARS)
train_model.summary()

Python
# Step 3: Data Preprocessing for Handwritten Images

def preprocess_image(image_path, img_width=256, img_height=64):
    """
    Preprocess handwritten Devanagari word image.
    Steps: Grayscale → Resize → Normalize → Invert (white text on black)
    """
    # Read image
    img = tf.io.read_file(image_path)
    img = tf.image.decode_png(img, channels=1)
    
    # Resize preserving aspect ratio (pad to target size)
    img = tf.image.resize_with_pad(img, img_height, img_width)
    
    # Normalize to [0, 1]
    img = tf.cast(img, tf.float32) / 255.0
    
    # Invert (handwriting is dark on light background)
    img = 1.0 - img
    
    return img

def encode_label(text, char_to_idx, max_len):
    """Convert Devanagari text to integer sequence."""
    encoded = [char_to_idx.get(c, 0) for c in text[:max_len]]
    # Pad with zeros
    encoded += [0] * (max_len - len(encoded))
    return np.array(encoded, dtype=np.int32)

def decode_prediction(pred, idx_to_char):
    """
    CTC greedy decoding: 
    1. Take argmax at each timestep
    2. Remove consecutive duplicates
    3. Remove blank tokens
    """
    # Greedy decode
    pred_indices = np.argmax(pred, axis=-1)
    
    # Remove consecutive duplicates
    decoded = []
    prev = -1
    for idx in pred_indices:
        if idx != prev and idx != len(idx_to_char):  # Not blank
            if idx in idx_to_char:
                decoded.append(idx_to_char[idx])
        prev = idx
    
    return ''.join(decoded)

# Example
sample_text = "नमस्ते"
encoded = encode_label(sample_text, char_to_idx, MAX_LABEL_LEN)
print(f"Text: {sample_text}")
print(f"Encoded: {encoded[:10]}...")

Python
# Step 4: Training

# Compile with dummy loss (CTC loss is built into model)
train_model.compile(
    optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),
    loss=lambda y_true, y_pred: y_pred  # CTC loss already computed
)

callbacks = [
    tf.keras.callbacks.EarlyStopping(
        monitor='val_loss', patience=10,
        restore_best_weights=True
    ),
    tf.keras.callbacks.ReduceLROnPlateau(
        monitor='val_loss', factor=0.5, patience=5
    ),
    tf.keras.callbacks.ModelCheckpoint(
        'best_devanagari_ocr.keras',
        monitor='val_loss', save_best_only=True
    ),
]

# Note: train_data and val_data should be tf.data.Dataset
# yielding ((image_batch, label_batch), dummy_output)
# history = train_model.fit(
#     train_data,
#     validation_data=val_data,
#     epochs=50,
#     callbacks=callbacks
# )

Python
# Step 5: Evaluation — Character Error Rate (CER) & Word Error Rate (WER)

def character_error_rate(y_true_texts, y_pred_texts):
    """
    CER = Edit_Distance(predicted, ground_truth) / len(ground_truth)
    Standard metric for OCR evaluation.
    """
    total_chars = 0
    total_errors = 0
    
    for true, pred in zip(y_true_texts, y_pred_texts):
        # Levenshtein distance
        distance = levenshtein_distance(true, pred)
        total_errors += distance
        total_chars += len(true)
    
    return total_errors / total_chars if total_chars > 0 else 0

def levenshtein_distance(s1, s2):
    """Compute edit distance between two strings."""
    if len(s1) < len(s2):
        return levenshtein_distance(s2, s1)
    
    if len(s2) == 0:
        return len(s1)
    
    prev_row = range(len(s2) + 1)
    for i, c1 in enumerate(s1):
        curr_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = prev_row[j + 1] + 1
            deletions = curr_row[j] + 1
            substitutions = prev_row[j] + (c1 != c2)
            curr_row.append(min(insertions, deletions, substitutions))
        prev_row = curr_row
    
    return prev_row[-1]

# Example evaluation
true_texts = ["नमस्ते", "भारत", "शिक्षा"]
pred_texts = ["नमस्ते", "भारत", "शिक्षो"]  # Last one has error

cer = character_error_rate(true_texts, pred_texts)
print(f"Character Error Rate (CER): {cer:.4f} ({cer*100:.1f}%)")

Improvement Suggestions

• Attention mechanism: Replace CTC with Transformer-based attention decoder for better accuracy on long words
• Language model: Post-process with a Hindi n-gram language model to correct OCR errors
• Multi-script: Extend to Bengali, Tamil, Telugu — share CNN backbone, train separate decoders
• Line segmentation: Add a text line detection step (CRAFT or DBNet) before word recognition
• Synthetic data: Generate training data using Hindi fonts with random distortions to augment real handwriting