Merge branch 'aditizyy:main' into main

README.md

@@ -3,7 +3,7 @@
 Generate personalized MCQs, short answer, and true/false questions using Hugging Face Transformers and a Streamlit UI.
 
 ## 💡 Features
-- T5-based Question generator (MCQ, short answer, true/false)
+- Question generator (MCQ, short answer, true/false)
 - Streamlit-based frontend
 - Cosine Similarity, BLEU-1, ROUGE -1 AND ROUGE-L Evaluation
 - Fine-tuned FLAN-T5 integration
@@ -45,7 +45,7 @@ custom-quiz-generator/
 ├── outputs/                        # Stores generated questions/outputs
 ├── valhalla/                       # T5-based fine-tuned models
 ├── requirements.txt                # Project dependencies
-├── FineTuneAndEvaluationscores_CLEANED.ipynb  # Evaluation notebook
+├── FineTuneAndEvaluationscores.ipynb  # Evaluation notebook
 ├── README.md                       # Project documentation
 └── .gitignore                      # Git ignore rules
-```
+```

deepset/roberta-base-squad2

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+"""Untitled7.ipynb
+Automatically generated by Colab.
+Original file is located at
+    https://colab.research.google.com/drive/1MWc3B3JSbW5VvEuftDi2WoCjUWN1CtVj
+"""
+pip install transformers datasets evaluate accelerate
+data_files = {
+    "train": "./train.json",  # If saved in current working directory
+    "validation": "./validation.json"
+}
+from google.colab import files
+uploaded = files.upload()  # Select and upload your train.json and validation.json files
+from google.colab import files
+uploaded = files.upload()  # Select and upload your train.json and validation.json files
+import json
+import pandas as pd
+from datasets import Dataset, DatasetDict
+with open("train.json", "r") as f:
+    train_data = json.load(f)
+with open("validation.json", "r") as f:
+    validation_data = json.load(f)
+train_list = train_data.get("data", [])
+validation_list = validation_data.get("data", [])
+train_df = pd.DataFrame(train_list)
+validation_df = pd.DataFrame(validation_list)
+train_dataset = Dataset.from_pandas(train_df)
+validation_dataset = Dataset.from_pandas(validation_df)
+dataset = DatasetDict({
+    "train": train_dataset,
+    "validation": validation_dataset
+})
+print(dataset)
+from transformers import AutoTokenizer, AutoModelForQuestionAnswering
+model_checkpoint = "deepset/roberta-base-squad2"
+tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
+model = AutoModelForQuestionAnswering.from_pretrained(model_checkpoint)
+def prepare_features(examples):
+    tokenized_examples = {
+        "input_ids": [],
+        "attention_mask": [],
+        "offset_mapping": [],
+        "overflow_to_sample_mapping": [],
+        "start_positions": [],
+        "end_positions": [],
+        "example_id": [], # Add example_id to link back to original examples
+    }
+
+    for example_index, paragraphs in enumerate(examples["paragraphs"]):
+        for para in paragraphs:
+            context = para["context"]
+            for qa in para["qas"]:
+                question = qa["question"]
+                answers = qa["answers"] # This is a list of answer dictionaries
+                tokenized = tokenizer(
+                    question,
+                    context,
+                    truncation="only_second",
+                    max_length=384,
+                    stride=128,
+                    return_overflowing_tokens=True,
+                    return_offsets_mapping=True,
+                    padding="max_length"
+                )
+                sample_mapping = tokenized.pop("overflow_to_sample_mapping")
+                offset_mapping = tokenized.pop("offset_mapping")
+                for i, offsets in enumerate(offset_mapping):
+                    input_ids = tokenized["input_ids"][i]
+                    cls_index = input_ids.index(tokenizer.cls_token_id)
+                    sequence_ids = tokenized.sequence_ids(i)
+                    start_position = cls_index
+                    end_position = cls_index
+                    if len(answers) > 0:
+                        first_answer = answers[0] # Get the first answer dictionary
+                        start_char = first_answer["answer_start"]
+                        end_char = start_char + len(first_answer["text"])
+                        token_start_index = 0
+                        while sequence_ids[token_start_index] != (1 if tokenizer.is_fast else 0):
+                            token_start_index += 1
+                        token_end_index = len(input_ids) - 1
+                        while sequence_ids[token_end_index] != (1 if tokenizer.is_fast else 0):
+                            token_end_index -= 1
+                        if offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char:
+                             # Move the token_start_index and token_end_index to the two ends of the answer
+                            while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:
+                                token_start_index += 1
+                            start_position = token_start_index - 1
+
+                            while token_end_index >= 0 and offsets[token_end_index][1] >= end_char:
+                                 token_end_index -= 1
+                            end_position = token_end_index + 1
+
+                    tokenized_examples["input_ids"].append(input_ids)
+                    tokenized_examples["attention_mask"].append(tokenized["attention_mask"][i])
+                    tokenized_examples["offset_mapping"].append(offsets)
+                    tokenized_examples["overflow_to_sample_mapping"].append(example_index) # Map back to the original example index in the batch
+                    tokenized_examples["start_positions"].append(start_position)
+                    tokenized_examples["end_positions"].append(end_position)
+                    tokenized_examples["example_id"].append(qa.get("id", f"{examples.get('title', ['no_title'])[example_index]}_{len(tokenized_examples['input_ids'])}"))
+tokenized_dataset = dataset.map(
+    prepare_features,
+    batched=True,
+    remove_columns=dataset["train"].column_names # Remove original columns after processing
+)
+print(tokenized_dataset)
+from transformers import TrainingArguments, Trainer
+training_args = TrainingArguments(
+    output_dir="./finetuned-roberta-squad2",
+    eval_strategy="epoch", # Corrected argument name
+    save_strategy="epoch", # Match save strategy to evaluation strategy
+    learning_rate=2e-5,
+    num_train_epochs=3,
+    weight_decay=0.01,
+    per_device_train_batch_size=8,
+    per_device_eval_batch_size=8,
+    save_total_limit=1,
+    load_best_model_at_end=True,
+)
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_dataset["train"],
+    eval_dataset=tokenized_dataset["validation"],
+    tokenizer=tokenizer
+)
+trainer.train()
+trainer.save_model("./finetuned-roberta-squad2")
+tokenizer.save_pretrained("./finetuned-roberta-squad2")
+# EVALUATION
+!pip install bert-score -q
+from transformers import pipeline
+qa_pipeline = pipeline("question-answering", model="./finetuned-roberta-squad2", tokenizer=tokenizer)
+examples = dataset["validation"]
+predictions = []
+references = []
+for example in examples:
+    for para in example["paragraphs"]:
+        context = para["context"]
+        for qa in para["qas"]:
+            question = qa["question"]
+            answers = qa["answers"] # This is a list of answer dictionaries
+            result = qa_pipeline({
+                "context": context,
+                "question": question
+            })
+            predictions.append(result["answer"])
+            if len(answers) > 0:
+                 references.append(answers[0]["text"])
+            else:
+                 references.append("") # Append empty string for unanswerable questions
+from bert_score import score
+P, R, F1 = score(predictions, references, lang="en", model_type="roberta-base")
+print(f"🔹 BERTScore Precision: {P.mean().item():.4f}")
+print(f"🔹 BERTScore Recall:    {R.mean().item():.4f}")
+print(f"🔹 BERTScore F1:        {F1.mean().item():.4f}")
+from transformers import AutoModel, AutoTokenizer
+import torch
+import torch.nn.functional as F
+# Use sentence transformer or same QA model encoder
+embed_model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+embed_tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+def get_embedding(text):
+    inputs = embed_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
+    with torch.no_grad():
+        outputs = embed_model(**inputs)
+    return outputs.last_hidden_state.mean(dim=1)
+# Compute cosine similarities
+cosine_scores = []
+for pred, ref in zip(predictions, references):
+    pred_emb = get_embedding(pred)
+    ref_emb = get_embedding(ref)
+    cosine_sim = F.cosine_similarity(pred_emb, ref_emb).item()
+    cosine_scores.append(cosine_sim)
+avg_cosine = sum(cosine_scores) / len(cosine_scores)
+print(f"🔹 Average Cosine Similarity: {avg_cosine:.4f}")

sentence-transformers/all-mpnet-base-v2

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""Untitled8.ipynb
+Automatically generated by Colab.
+Original file is located at
+    https://colab.research.google.com/drive/1JMKmuuP0equrOr6l6oQVQbpbBnGTGvcc
+"""
+!pip install sentence-transformers
+from google.colab import files
+import pandas as pd
+import random
+uploaded = files.upload()
+file_name = list(uploaded.keys())[0]
+df = pd.read_csv(file_name)
+
+# Preview
+print("📄 Preview of training data:")
+print(df.head())
+print(f"\n✅ Loaded {len(df)} training pairs.")
+from sentence_transformers import InputExample
+train_examples = [
+    InputExample(texts=[row["text1"], row["text2"]], label=float(row["score"]))
+    for _, row in df.iterrows()
+]
+from sentence_transformers import SentenceTransformer, losses
+from torch.utils.data import DataLoader
+model = SentenceTransformer("sentence-transformers/all-mpnet-base-v2")
+train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=16)
+train_loss = losses.CosineSimilarityLoss(model)
+model.fit(
+    train_objectives=[(train_dataloader, train_loss)],
+    epochs=1,                # Increase to 3–5 for better results
+    warmup_steps=10,         # Usually 10% of steps per epoch
+    output_path="fine-tuned-mpnet-model"
+)
+from sentence_transformers import SentenceTransformer
+model = SentenceTransformer("fine-tuned-mpnet-model")
+sentence = "This is a test sentence."
+embedding = model.encode(sentence)
+print(embedding.shape)
+fine_tuned_model = SentenceTransformer("fine-tuned-mpnet-model")
+
+# Example usage
+embedding = fine_tuned_model.encode("This is a test sentence.")
+print("🔢 Embedding shape:", embedding.shape)
+import os
+print(os.listdir("fine-tuned-mpnet-model"))
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import cos_sim
+# Load base and fine-tuned models
+base_model = SentenceTransformer("sentence-transformers/all-mpnet-base-v2")
+ft_model = SentenceTransformer("fine-tuned-mpnet-model")
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator
+from torch.utils.data import DataLoader
+fine_tuned_model = SentenceTransformer("fine-tuned-mpnet-model")
+sentence = "This is a test sentence."
+embedding = fine_tuned_model.encode(sentence)
+print("🔢 Embedding shape:", embedding.shape)
+
+"""You can now use the `fine_tuned_model` to generate embeddings for any text data. For example, you can use these embeddings for tasks like semantic search, clustering, or classification."""
+
+from tqdm import tqdm
+import numpy as np
+from sentence_transformers.util import cos_sim
+from sklearn.metrics import mean_squared_error
+from scipy.stats import spearmanr
+def evaluate_model(model, name):
+    embeddings1 = model.encode(sentences1, convert_to_tensor=True)
+    embeddings2 = model.encode(sentences2, convert_to_tensor=True)
+    similarities = cos_sim(embeddings1, embeddings2).diagonal().cpu().numpy()
+    mse = mean_squared_error(true_scores, similarities)
+    spearman_corr, _ = spearmanr(true_scores, similarities)
+    print(f"\n📋 Evaluation: {name}")
+    print(f"📐 CosineSim vs Human Scores: ")
+    print(f"  • MSE:         {mse:.4f}")
+    print(f"  • Spearman R:  {spearman_corr:.4f}")
+    return similarities
+# Extract sentences and scores from the DataFrame
+sentences1 = df['text1'].tolist()
+sentences2 = df['text2'].tolist()
+true_scores = df['score'].tolist()
+# Evaluate both models
+_ = evaluate_model(base_model, "Base MPNET")
+_ = evaluate_model(ft_model, "Fine-Tuned MPNET")