Chapter 21: Applied Deep Learning — NLP and Sequence Projects

Problem Statement

Given a tweet or social media post written in Hindi, English, or Hinglish (code-mixed Hindi-English), classify it as Positive, Negative, or Neutral. The input may use Devanagari script, Roman script, or a mix of both.

Why This Is Hard

Consider these three tweets — all expressing negativity:

• "Worst movie ever" — Pure English. Easy.
• "बहुत बकवास फिल्म थी" — Pure Hindi (Devanagari). Needs Hindi-aware tokenizer.
• "Bahut bakwas movie thi 👎" — Hinglish (Roman). Neither English nor Hindi tokenizer handles this well.

Dataset

We use the SemEval 2020 Task 9 Hinglish Sentiment dataset and augment with:

Tokenization Challenges

from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
tokens = tokenizer.tokenize("बहुत अच्छी फिल्म")
print(tokens)
# Output: ['[UNK]']  ← Every Devanagari character is unknown!

tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
tokens = tokenizer.tokenize("बहुत अच्छी फिल्म")
print(tokens)
# Output: ['बहुत', 'अच्छी', 'फिल्म']  ← Works! But only ~1,700 Hindi tokens

tokens = tokenizer.tokenize("bahut acchi movie thi yaar")
print(tokens)
# Output: ['ba', '##hu', '##t', 'ac', '##chi', 'movie', 'th', '##i', 'ya', '##ar']
# "bahut" → 3 subwords, "acchi" → 2 subwords — heavy fragmentation!

Model Architecture

HuggingFace Implementation

Python
# Project 1: Hindi/Hinglish Sentiment Analysis with MuRIL
import torch
from torch.utils.data import Dataset, DataLoader
from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer
)
from sklearn.metrics import classification_report, f1_score
import pandas as pd
import re
import numpy as np

# ─── Step 1: Preprocessing for Hinglish ───
def preprocess_hinglish(text):
    """Clean Hinglish tweet text."""
    text = re.sub(r'http\S+', '', text)           # Remove URLs
    text = re.sub(r'@\w+', '', text)              # Remove mentions
    text = re.sub(r'#(\w+)', r'\1', text)        # Keep hashtag text
    text = re.sub(r'[^\w\s\u0900-\u097F]', '', text)  # Keep Devanagari + alphanumeric
    text = re.sub(r'\s+', ' ', text).strip()
    text = text.lower()
    # Normalize common Hinglish spelling variants
    spelling_map = {
        'bohot': 'bahut', 'boht': 'bahut', 'bhot': 'bahut',
        'accha': 'acha', 'acha': 'acha', 'achha': 'acha',
        'kya': 'kya', 'kyaa': 'kya', 'kia': 'kya',
    }
    words = text.split()
    words = [spelling_map.get(w, w) for w in words]
    return ' '.join(words)

# ─── Step 2: Dataset class ───
class HinglishDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_len=128):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_len = max_len

    def __len__(self):
        return len(self.texts)

    def __getitem__(self, idx):
        encoding = self.tokenizer(
            self.texts[idx],
            max_length=self.max_len,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        return {
            'input_ids': encoding['input_ids'].squeeze(),
            'attention_mask': encoding['attention_mask'].squeeze(),
            'labels': torch.tensor(self.labels[idx], dtype=torch.long)
        }

# ─── Step 3: Load model — MuRIL for Indian languages ───
MODEL_NAME = "google/muril-base-cased"
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = AutoModelForSequenceClassification.from_pretrained(
    MODEL_NAME, num_labels=3  # Positive, Negative, Neutral
)

# ─── Step 4: Prepare data ───
# (Assuming df has columns: 'text', 'label'  where label ∈ {0,1,2})
# df = pd.read_csv('hinglish_sentiment.csv')
# For demo, we create sample data:
sample_texts = [
    preprocess_hinglish("bahut acchi movie thi! loved it"),
    preprocess_hinglish("worst film ever dekhi maine"),
    preprocess_hinglish("ठीक ठाक थी, nothing special"),
    preprocess_hinglish("kya bakwas acting thi yaar"),
    preprocess_hinglish("mast movie hai bhai 🔥"),
]
sample_labels = [0, 1, 2, 1, 0]  # 0=Positive, 1=Negative, 2=Neutral

# ─── Step 5: Training configuration ───
training_args = TrainingArguments(
    output_dir='./hinglish_sentiment_model',
    num_train_epochs=4,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    learning_rate=2e-5,
    weight_decay=0.01,
    warmup_ratio=0.1,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    metric_for_best_model="f1_weighted",
    fp16=True,  # Mixed precision for faster training
)

# ─── Step 6: Custom metrics ───
def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    preds = np.argmax(predictions, axis=1)
    f1 = f1_score(labels, preds, average='weighted')
    return {'f1_weighted': f1}

# ─── Step 7: Train ───
train_dataset = HinglishDataset(sample_texts, sample_labels, tokenizer)
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=train_dataset,  # Use proper val split in practice!
    compute_metrics=compute_metrics,
)
# trainer.train()  # Uncomment to actually train

# ─── Step 8: Inference ───
def predict_sentiment(text):
    text = preprocess_hinglish(text)
    inputs = tokenizer(text, return_tensors="pt",
                       truncation=True, max_length=128)
    with torch.no_grad():
        outputs = model(**inputs)
    probs = torch.softmax(outputs.logits, dim=-1)
    label = ["Positive", "Negative", "Neutral"][probs.argmax()]
    return label, probs.tolist()

# Test it
print(predict_sentiment("bahut mast movie thi!"))
print(predict_sentiment("bakwas film, time waste"))
print(predict_sentiment("ठीक-ठाक थी"))

Expected Metrics

Error Analysis

Problem Statement

Given a full Indian Supreme Court or High Court judgment (typically 5,000–50,000 words), generate a concise abstractive summary (300–800 words) that captures the facts, issues, arguments, reasoning, and holding.

Why This Matters

Dataset

Tokenization Challenges for Legal Text

• Extreme length: A 10,000-word judgment exceeds BERT's 512-token limit by 20×. You need either a Longformer (4,096 tokens) or a chunking strategy.
• Legal vocabulary: Terms like "suo motu," "certiorari," "mandamus," "res judicata" are Latin loan-words used in Indian courts but absent from standard NLP vocabularies.
• Hindi legal terms: "याचिका" (petition), "अपील" (appeal), "निर्णय" (judgment) appear when courts cite Hindi-language proceedings.
• Section references: "Section 302 IPC," "Article 21," "CrPC 482" — these are not natural language but structured legal codes.

Model Architecture: Two Approaches

HuggingFace Implementation

Python
# Project 2: Indian Legal Document Summarization
import torch
from transformers import (
    AutoTokenizer, AutoModelForSeq2SeqLM,
    Seq2SeqTrainingArguments, Seq2SeqTrainer,
    DataCollatorForSeq2Seq
)
from datasets import load_dataset
import evaluate
import numpy as np
import re

# ─── Step 1: Legal text preprocessing ───
def preprocess_legal_text(text):
    """Clean Indian legal judgment text."""
    # Normalize section references
    text = re.sub(r'Section\s+(\d+)', r'Section_\1', text)
    text = re.sub(r'Article\s+(\d+)', r'Article_\1', text)
    # Remove page numbers and formatting artifacts
    text = re.sub(r'\n\d+\n', '\n', text)
    text = re.sub(r'\s+', ' ', text).strip()
    # Truncate to manageable length for encoder
    words = text.split()
    if len(words) > 3000:
        # Keep first 1500 + last 1500 words (intro + conclusion)
        words = words[:1500] + ['[...]'] + words[-1500:]
    return ' '.join(words)

# ─── Step 2: Extractive pre-filtering ───
from transformers import pipeline

def extractive_filter(text, top_k=30):
    """Use sentence scoring to extract key sentences."""
    sentences = text.split('.')
    # Simple heuristic: score sentences with legal keywords
    legal_keywords = [
        'held', 'order', 'appeal', 'petitioner', 'respondent',
        'court', 'judgment', 'section', 'article', 'contention',
        'submitted', 'evidence', 'guilty', 'acquit', 'convicted',
        'constitutional', 'fundamental', 'right', 'violation'
    ]
    scored = []
    for i, sent in enumerate(sentences):
        score = sum(1 for kw in legal_keywords if kw in sent.lower())
        # Boost first and last 20% of document (facts + holding)
        if i < len(sentences) * 0.2 or i > len(sentences) * 0.8:
            score += 2
        scored.append((score, sent))
    scored.sort(reverse=True)
    return '. '.join([s for _, s in scored[:top_k]])

# ─── Step 3: Load mT5 for multilingual summarization ───
MODEL_NAME = "google/mt5-base"  # or "ai4bharat/IndicBART"
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)

# ─── Step 4: Summarization function ───
def summarize_judgment(judgment_text, max_input=1024, max_output=256):
    # Stage 1: Extract key sentences
    filtered = extractive_filter(judgment_text)
    # Stage 2: Abstractive summarization
    prefix = "summarize: "
    inputs = tokenizer(
        prefix + filtered,
        max_length=max_input,
        truncation=True,
        return_tensors="pt"
    )
    summary_ids = model.generate(
        inputs["input_ids"],
        max_length=max_output,
        min_length=60,
        num_beams=4,
        length_penalty=2.0,
        no_repeat_ngram_size=3,
        early_stopping=True
    )
    summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)
    return summary

# ─── Step 5: Evaluation with ROUGE ───
rouge = evaluate.load("rouge")

def evaluate_summary(prediction, reference):
    results = rouge.compute(
        predictions=[prediction],
        references=[reference]
    )
    print(f"ROUGE-1: {results['rouge1']:.4f}")
    print(f"ROUGE-2: {results['rouge2']:.4f}")
    print(f"ROUGE-L: {results['rougeL']:.4f}")
    return results

# Example usage
sample_judgment = """
The petitioner filed a writ petition under Article 32 of the Constitution 
challenging the constitutional validity of Section 66A of the Information 
Technology Act, 2000. The petitioner contended that the provision was vague 
and overbroad, violating the fundamental right to free speech under 
Article 19(1)(a). The respondent Union of India argued that reasonable 
restrictions under Article 19(2) were applicable. After hearing both sides, 
this Court held that Section 66A is unconstitutional as it creates a 
chilling effect on free speech and is not saved by Article 19(2). 
The impugned section is struck down.
"""

summary = summarize_judgment(sample_judgment)
print("Generated Summary:", summary)

Metrics: Extractive vs. Abstractive on ILDC

Error Analysis

Problem Statement

Build a conversational AI system that handles Indian Railways booking queries in Hindi, English, and Hinglish. The system must understand queries like:

• "Delhi se Mumbai ka train kitne baje hai?" (What time is the train from Delhi to Mumbai?)
• "मुझे कल दिल्ली से जयपुर जाना है" (I need to go from Delhi to Jaipur tomorrow)
• "PNR status check karo 4521876543" (Check PNR status for 4521876543)
• "Cancel my ticket from last week"

Architecture: Intent + Slot Filling + Response Generation

Part A: Intent Classification

Python
# Part A: Intent classification for IRCTC queries
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

# Define intents
INTENTS = [
    'SEARCH_TRAIN',       # Find trains between stations
    'CHECK_PNR',          # PNR status inquiry
    'BOOK_TICKET',        # Book a ticket
    'CANCEL_TICKET',      # Cancel booking
    'SEAT_AVAILABILITY',  # Check seat availability
    'TRAIN_STATUS',       # Live running status
    'GENERAL_QUERY',      # General help
]

# Training data examples (would be 1000+ in practice)
training_examples = [
    ("Delhi se Mumbai train dikhao", 0),
    ("दिल्ली से मुंबई की ट्रेन बताओ", 0),
    ("Show me trains from Delhi to Mumbai", 0),
    ("PNR check karo 4521876543", 1),
    ("mera PNR status kya hai", 1),
    ("ticket book karna hai kal ke liye", 2),
    ("मुझे टिकट बुक करनी है", 2),
    ("cancel my ticket please", 3),
    ("meri ticket cancel karo", 3),
    ("kitni seat available hai?", 4),
    ("train kahan tak pahunchi?", 5),
    ("help chahiye", 6),
]

# Load MuRIL for intent classification
model_name = "google/muril-base-cased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
intent_model = AutoModelForSequenceClassification.from_pretrained(
    model_name, num_labels=len(INTENTS)
)

def classify_intent(query):
    inputs = tokenizer(query, return_tensors="pt",
                       truncation=True, max_length=64)
    with torch.no_grad():
        logits = intent_model(**inputs).logits
    pred = logits.argmax(dim=-1).item()
    confidence = torch.softmax(logits, dim=-1).max().item()
    return INTENTS[pred], confidence

Part B: Slot Extraction (Named Entity Recognition)

Python
# Part B: Slot filling using token classification
from transformers import AutoModelForTokenClassification

# Slot labels (BIO format)
SLOT_LABELS = [
    'O',          # Outside
    'B-SOURCE',   # Source station start
    'I-SOURCE',   # Source station continuation
    'B-DEST',     # Destination start
    'I-DEST',     # Destination continuation
    'B-DATE',     # Date start
    'I-DATE',     # Date continuation
    'B-PNR',      # PNR number
    'I-PNR',
    'B-CLASS',    # Travel class (Sleeper, AC, etc.)
    'I-CLASS',
    'B-TRAIN',    # Train name/number
    'I-TRAIN',
]

# Example annotation
# "Delhi  se  Mumbai  3  tareekh  ko  AC  mein  dikhao"
# B-SRC  O   B-DEST  B-DATE I-DATE O  B-CLS O    O

slot_model = AutoModelForTokenClassification.from_pretrained(
    model_name, num_labels=len(SLOT_LABELS)
)

def extract_slots(query):
    inputs = tokenizer(query, return_tensors="pt",
                       truncation=True, max_length=64,
                       return_offsets_mapping=True)
    offsets = inputs.pop("offset_mapping")
    with torch.no_grad():
        logits = slot_model(**inputs).logits
    preds = logits.argmax(dim=-1)[0].tolist()

    slots = {}
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
    current_slot = None
    current_value = []

    for token, pred_id in zip(tokens, preds):
        label = SLOT_LABELS[pred_id]
        if label.startswith('B-'):
            if current_slot:
                slots[current_slot] = ' '.join(current_value)
            current_slot = label[2:]
            current_value = [token]
        elif label.startswith('I-') and current_slot:
            current_value.append(token)
        else:
            if current_slot:
                slots[current_slot] = ' '.join(current_value)
            current_slot = None
            current_value = []

    if current_slot:
        slots[current_slot] = ' '.join(current_value)
    return slots

# ─── Part C: Response Generation ───
def generate_response(intent, slots):
    """Template-based response with bilingual output."""
    if intent == 'SEARCH_TRAIN':
        src = slots.get('SOURCE', '?')
        dst = slots.get('DEST', '?')
        date = slots.get('DATE', 'today')
        return (f"{src} → {dst} ke liye {date} ko trains:\n"
                f"1. Rajdhani Express (12:05 PM)\n"
                f"2. Duronto Express (11:20 PM)\n"
                f"3. Mumbai Mail (09:00 PM)")
    elif intent == 'CHECK_PNR':
        pnr = slots.get('PNR', 'N/A')
        return f"PNR {pnr} ka status: Confirmed (S5, Berth 42)"
    else:
        return "Main aapki kaise madad kar sakta hoon?"

# ─── Full pipeline ───
def chatbot_pipeline(user_query):
    intent, confidence = classify_intent(user_query)
    slots = extract_slots(user_query)
    response = generate_response(intent, slots)
    return {
        'query': user_query,
        'intent': intent,
        'confidence': confidence,
        'slots': slots,
        'response': response
    }

# Test
result = chatbot_pipeline("Delhi se Mumbai kal AC mein train dikhao")
print(result)

Metrics

def extract_pnr(query):
    import re
    # Bug 1: Pattern issue
    pnr_pattern = r'\d{5}'    # PNR is 10 digits!
    match = re.search(pnr_pattern, query)
    if match:
        # Bug 2: Wrong method
        return match.groups()  # .group() not .groups()
    # Bug 3: Missing return
    # If no match, function returns None implicitly
    # Should return empty string or raise exception

result = extract_pnr("PNR check karo 4521876543")
print("PNR:", result)  # Gives wrong output!

Problem Statement

Given a message forwarded on WhatsApp (in Hindi, English, or Hinglish), classify it as REAL or FAKE. The model must handle health misinformation, political propaganda, communal rumors, and scientific hoaxes common in Indian WhatsApp groups.

Why WhatsApp Fake News Is Unique to India

Dataset

Feature Engineering for WhatsApp Text

HuggingFace Implementation

Python
# Project 4: Fake News Detection for Indian WhatsApp Messages
import torch
import torch.nn as nn
from transformers import AutoTokenizer, AutoModel
from sklearn.metrics import classification_report
import re
import numpy as np

# ─── Step 1: Feature extraction ───
def extract_meta_features(text):
    """Extract non-textual features that signal fakeness."""
    features = []
    # Exclamation density
    features.append(text.count('!') / max(len(text), 1))
    # CAPS ratio
    caps = sum(1 for c in text if c.isupper())
    features.append(caps / max(len(text), 1))
    # Has source citation
    source_words = ['according', 'source', 'study', 'report',
                    'research', 'PTI', 'ANI', 'WHO']
    features.append(1.0 if any(w in text.lower() for w in source_words) else 0.0)
    # Forward indicators
    fwd_words = ['forward', 'share', 'viral', 'forwarded',
                 'भेजो', 'शेयर', 'forward करो']
    features.append(1.0 if any(w in text.lower() for w in fwd_words) else 0.0)
    # Message length (normalized)
    features.append(min(len(text) / 1000, 1.0))
    return features

# ─── Step 2: BERT + Meta Features Model ───
class FakeNewsDetector(nn.Module):
    def __init__(self, bert_model_name, n_meta_features=5, n_classes=2):
        super().__init__()
        self.bert = AutoModel.from_pretrained(bert_model_name)
        self.dropout = nn.Dropout(0.3)
        # BERT hidden (768) + meta features (5) → classifier
        self.classifier = nn.Sequential(
            nn.Linear(768 + n_meta_features, 256),
            nn.ReLU(),
            nn.Dropout(0.2),
            nn.Linear(256, n_classes)
        )

    def forward(self, input_ids, attention_mask, meta_features):
        bert_out = self.bert(
            input_ids=input_ids,
            attention_mask=attention_mask
        )
        cls_embedding = bert_out.last_hidden_state[:, 0, :]  # [CLS] token
        cls_embedding = self.dropout(cls_embedding)
        # Concatenate BERT [CLS] with meta features
        combined = torch.cat([cls_embedding, meta_features], dim=1)
        logits = self.classifier(combined)
        return logits

# ─── Step 3: Initialize ───
MODEL_NAME = "google/muril-base-cased"
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = FakeNewsDetector(MODEL_NAME)

# ─── Step 4: Inference function ───
def detect_fake_news(text):
    # Tokenize
    encoding = tokenizer(
        text, max_length=256, padding='max_length',
        truncation=True, return_tensors='pt'
    )
    # Extract meta features
    meta = torch.tensor([extract_meta_features(text)], dtype=torch.float)

    model.eval()
    with torch.no_grad():
        logits = model(
            encoding['input_ids'],
            encoding['attention_mask'],
            meta
        )
    probs = torch.softmax(logits, dim=-1)
    pred = probs.argmax(dim=-1).item()
    label = "FAKE" if pred == 1 else "REAL"
    return label, probs[0].tolist()

# Test
test_messages = [
    "BREAKING: गर्म पानी पीने से COVID ठीक होता है!!! SHARE करो सबको!!!",
    "According to WHO, COVID-19 vaccines have undergone rigorous clinical trials.",
    "NASA ne confirm kiya hai ki kal raat 2 baje earthquake aayega India mein!!!",
]
for msg in test_messages:
    label, probs = detect_fake_news(msg)
    print(f"[{label}] (conf: {max(probs):.2f}) → {msg[:50]}...")

Expected Metrics

Error Analysis

Problem Statement

Build a Hindi→English and English→Hindi translation system using the Transformer architecture. Implement attention visualization to understand word alignment, and evaluate using BLEU score.

Mathematical Foundation: BLEU Score

pₙ = Σ_sentence Σ_ngram min(count_candidate(ngram), count_reference(ngram))
     ────────────────────────────────────────────────────────────────────────
     Σ_sentence Σ_ngram count_candidate(ngram)

BP = exp(1 - r/c)  if c < r
BP = 1              if c ≥ r

where c = candidate length, r = reference length

BLEU = BP · exp(Σₙ wₙ · log(pₙ))

Typically: w₁ = w₂ = w₃ = w₄ = 0.25  (equal weights)

Tokenization: SentencePiece for Hindi

Python
# Why we need SentencePiece for Hindi
# Hindi has agglutinative properties — words combine morphemes
# "खाऊँगी" = खा (eat) + ऊँगी (will, feminine) → "I will eat" (said by a woman)

import sentencepiece as spm

# Train SentencePiece model on Hindi corpus
# spm.SentencePieceTrainer.train(
#     input='hindi_corpus.txt',
#     model_prefix='hindi_sp',
#     vocab_size=32000,
#     character_coverage=0.9995,  # High coverage for Devanagari
#     model_type='bpe'
# )

# Compare tokenization approaches
from transformers import AutoTokenizer

sentence = "भारत के प्रधानमंत्री ने आज एक महत्वपूर्ण बैठक की।"

# mBERT tokenizer (WordPiece)
tok1 = AutoTokenizer.from_pretrained("bert-base-multilingual-cased")
print("mBERT:", tok1.tokenize(sentence))
# ['भारत', 'के', 'प्रधानमंत्री', 'ने', 'आज', 'एक', 'महत्व', '##पूर्ण', 'बैठक', 'की', '।']
# Note: "महत्वपूर्ण" (important) split into "महत्व" + "##पूर्ण"

# IndicTrans tokenizer (SentencePiece)
tok2 = AutoTokenizer.from_pretrained("ai4bharat/indictrans2-en-indic-1B")
print("IndicTrans:", tok2.tokenize(sentence))
# Better handling of compound Hindi words

Full Implementation: Hindi↔English NMT

Python
# Project 5: Hindi↔English Neural Machine Translation
from transformers import (
    AutoTokenizer, AutoModelForSeq2SeqLM,
    Seq2SeqTrainingArguments, Seq2SeqTrainer,
    DataCollatorForSeq2Seq
)
from datasets import load_dataset
import evaluate
import numpy as np
import torch

# ─── Step 1: Load IndicTrans2 (State-of-the-art for Indian languages) ───
MODEL_NAME = "ai4bharat/indictrans2-en-indic-1B"
# For a lighter model, use Helsinki-NLP/opus-mt-en-hi
LITE_MODEL = "Helsinki-NLP/opus-mt-en-hi"

tokenizer = AutoTokenizer.from_pretrained(LITE_MODEL)
model = AutoModelForSeq2SeqLM.from_pretrained(LITE_MODEL)

# ─── Step 2: Translation function ───
def translate_en_to_hi(text, max_length=128):
    inputs = tokenizer(text, return_tensors="pt",
                       max_length=max_length, truncation=True)
    translated = model.generate(
        **inputs,
        max_length=max_length,
        num_beams=5,
        length_penalty=1.0,
        no_repeat_ngram_size=2,
        early_stopping=True
    )
    result = tokenizer.decode(translated[0], skip_special_tokens=True)
    return result

# ─── Step 3: BLEU Score Computation ───
import sacrebleu

def compute_bleu(predictions, references):
    """Compute BLEU score using sacrebleu."""
    bleu = sacrebleu.corpus_bleu(predictions, [references])
    print(f"BLEU Score: {bleu.score:.2f}")
    print(f"Precisions: {[f'{p:.1f}' for p in bleu.precisions]}")
    print(f"Brevity Penalty: {bleu.bp:.4f}")
    print(f"Length Ratio: {bleu.sys_len}/{bleu.ref_len} = {bleu.sys_len/bleu.ref_len:.3f}")
    return bleu

# ─── Step 4: By-hand BLEU calculation (for understanding) ───
from collections import Counter

def bleu_from_scratch(candidate, reference, max_n=4):
    """Compute BLEU score from scratch — see the math in action."""
    cand_tokens = candidate.split()
    ref_tokens = reference.split()

    # Step 1: Compute clipped n-gram precisions
    precisions = []
    for n in range(1, max_n + 1):
        # Extract n-grams
        cand_ngrams = [tuple(cand_tokens[i:i+n]) for i in range(len(cand_tokens) - n + 1)]
        ref_ngrams = [tuple(ref_tokens[i:i+n]) for i in range(len(ref_tokens) - n + 1)]

        # Count
        cand_counts = Counter(cand_ngrams)
        ref_counts = Counter(ref_ngrams)

        # Clipped count
        clipped = 0
        total = 0
        for ngram, count in cand_counts.items():
            clipped += min(count, ref_counts.get(ngram, 0))
            total += count

        precision = clipped / max(total, 1)
        precisions.append(precision)
        print(f"  p{n} = {clipped}/{total} = {precision:.4f}")

    # Step 2: Brevity penalty
    import math
    c = len(cand_tokens)
    r = len(ref_tokens)
    bp = math.exp(1 - r/c) if c < r else 1.0
    print(f"  BP = {bp:.4f} (candidate: {c}, reference: {r})")

    # Step 3: BLEU = BP × geometric mean of precisions
    log_avg = sum(math.log(max(p, 1e-10)) for p in precisions) / max_n
    bleu = bp * math.exp(log_avg)
    print(f"  BLEU = {bleu:.4f}")
    return bleu

# Example
ref = "India is a diverse country with many languages"
cand = "India is a country with many diverse languages"
print("=== By-hand BLEU ===")
bleu_from_scratch(cand, ref)

# ─── Step 5: Attention Visualization ───
def visualize_attention(source, target_prefix=""):
    """Extract and visualize cross-attention weights."""
    inputs = tokenizer(source, return_tensors="pt")

    # Generate with attention outputs
    outputs = model.generate(
        **inputs,
        max_length=64,
        num_beams=1,
        output_attentions=True,
        return_dict_in_generate=True,
    )

    # Decode tokens
    src_tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
    tgt_tokens = tokenizer.convert_ids_to_tokens(outputs.sequences[0])

    print(f"Source:  {' '.join(src_tokens)}")
    print(f"Target:  {' '.join(tgt_tokens)}")

    # In practice, you'd use matplotlib heatmap here
    # plt.imshow(attention_matrix, cmap='viridis')
    # plt.xticks(range(len(src_tokens)), src_tokens, rotation=45)
    # plt.yticks(range(len(tgt_tokens)), tgt_tokens)
    return src_tokens, tgt_tokens

# Test translations
test_pairs = [
    ("The Supreme Court upheld the fundamental right to privacy.",
     "सर्वोच्च न्यायालय ने निजता के मौलिक अधिकार को बरकरार रखा।"),
    ("Please book a ticket from Delhi to Mumbai.",
     "कृपया दिल्ली से मुंबई तक का टिकट बुक करें।"),
    ("Artificial intelligence is transforming healthcare in India.",
     "कृत्रिम बुद्धिमत्ता भारत में स्वास्थ्य सेवा को बदल रही है।"),
]

for en, hi in test_pairs:
    predicted_hi = translate_en_to_hi(en)
    print(f"EN: {en}")
    print(f"HI (pred): {predicted_hi}")
    print(f"HI (ref):  {hi}")
    print()

Attention Visualization (ASCII)

Expected Metrics

Error Analysis: Where Hindi NMT Struggles