Upload 2 files

evaluation_module.py

@@ -0,0 +1,229 @@
+'''import torch
+from sacrebleu import corpus_bleu
+from rouge_score import rouge_scorer
+from bert_score import score
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, pipeline
+from transformers import AutoModelForSequenceClassification
+import nltk
+from nltk.util import ngrams
+from nltk.tokenize import word_tokenize
+from nltk.translate.meteor_score import meteor_score
+from nltk.translate.chrf_score import sentence_chrf
+from textstat import flesch_reading_ease, flesch_kincaid_grade
+from sklearn.metrics.pairwise import cosine_similarity
+
+class RAGEvaluator:
+    def __init__(self):
+        self.gpt2_model, self.gpt2_tokenizer = self.load_gpt2_model()
+        self.bias_pipeline = pipeline("zero-shot-classification", model="Hate-speech-CNERG/dehatebert-mono-english")
+
+    def load_gpt2_model(self):
+        model = GPT2LMHeadModel.from_pretrained('gpt2')
+        tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+        return model, tokenizer
+
+    def evaluate_bleu_rouge(self, candidates, references):
+        bleu_score = corpus_bleu(candidates, [references]).score
+        scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
+        rouge_scores = [scorer.score(ref, cand) for ref, cand in zip(references, candidates)]
+        rouge1 = sum([score['rouge1'].fmeasure for score in rouge_scores]) / len(rouge_scores)
+        return bleu_score, rouge1
+
+    def evaluate_bert_score(self, candidates, references):
+        P, R, F1 = score(candidates, references, lang="en", model_type='bert-base-multilingual-cased')
+        return P.mean().item(), R.mean().item(), F1.mean().item()
+
+    def evaluate_perplexity(self, text):
+        encodings = self.gpt2_tokenizer(text, return_tensors='pt')
+        max_length = self.gpt2_model.config.n_positions
+        stride = 512
+        lls = []
+        for i in range(0, encodings.input_ids.size(1), stride):
+            begin_loc = max(i + stride - max_length, 0)
+            end_loc = min(i + stride, encodings.input_ids.size(1))
+            trg_len = end_loc - i
+            input_ids = encodings.input_ids[:, begin_loc:end_loc]
+            target_ids = input_ids.clone()
+            target_ids[:, :-trg_len] = -100
+            with torch.no_grad():
+                outputs = self.gpt2_model(input_ids, labels=target_ids)
+                log_likelihood = outputs[0] * trg_len
+            lls.append(log_likelihood)
+        ppl = torch.exp(torch.stack(lls).sum() / end_loc)
+        return ppl.item()
+
+    def evaluate_diversity(self, texts):
+        all_tokens = [tok for text in texts for tok in text.split()]
+        unique_bigrams = set(ngrams(all_tokens, 2))
+        diversity_score = len(unique_bigrams) / len(all_tokens) if all_tokens else 0
+        return diversity_score
+
+    def evaluate_racial_bias(self, text):
+        results = self.bias_pipeline([text], candidate_labels=["hate speech", "not hate speech"])
+        bias_score = results[0]['scores'][results[0]['labels'].index('hate speech')]
+        return bias_score
+
+    def evaluate_meteor(self, candidates, references):
+        nltk.download('punkt', quiet=True)  
+        
+        meteor_scores = [
+            meteor_score([word_tokenize(ref)], word_tokenize(cand))
+            for ref, cand in zip(references, candidates)
+        ]
+        return sum(meteor_scores) / len(meteor_scores)
+    
+    def evaluate_chrf(self, candidates, references):
+        chrf_scores = [sentence_chrf(ref, cand) for ref, cand in zip(references, candidates)]
+        return sum(chrf_scores) / len(chrf_scores)
+    
+    def evaluate_readability(self, text):
+        flesch_ease = flesch_reading_ease(text)
+        flesch_grade = flesch_kincaid_grade(text)
+        return flesch_ease, flesch_grade
+
+    def evaluate_all(self, response, reference):
+        candidates = [response]
+        references = [reference]
+        bleu, rouge1 = self.evaluate_bleu_rouge(candidates, references)
+        bert_p, bert_r, bert_f1 = self.evaluate_bert_score(candidates, references)
+        perplexity = self.evaluate_perplexity(response)
+        diversity = self.evaluate_diversity(candidates)
+        racial_bias = self.evaluate_racial_bias(response)
+        meteor = self.evaluate_meteor(candidates, references)
+        chrf = self.evaluate_chrf(candidates, references)
+        flesch_ease, flesch_grade = self.evaluate_readability(response)
+        return {
+            "BLEU": bleu,
+            "ROUGE-1": rouge1,
+            "BERT P": bert_p,
+            "BERT R": bert_r,
+            "BERT F1": bert_f1,
+            "Perplexity": perplexity,
+            "Diversity": diversity,
+            "Racial Bias": racial_bias,
+            "METEOR": meteor,
+            "CHRF": chrf,
+            "Flesch Reading Ease": flesch_ease,
+            "Flesch-Kincaid Grade": flesch_grade,
+        }'''
+
+
+import torch
+from sacrebleu import corpus_bleu
+from rouge_score import rouge_scorer
+from bert_score import score
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, pipeline, AutoModelForSequenceClassification, AutoTokenizer
+import nltk
+from nltk.util import ngrams
+from nltk.tokenize import word_tokenize
+from nltk.translate.meteor_score import meteor_score
+from nltk.translate.chrf_score import sentence_chrf
+from textstat import flesch_reading_ease, flesch_kincaid_grade
+from sklearn.metrics.pairwise import cosine_similarity
+
+class RAGEvaluator:
+    def __init__(self):
+        self.gpt2_model, self.gpt2_tokenizer = self.load_gpt2_model()
+        self.bias_pipeline = self.load_bias_model()
+
+    def load_gpt2_model(self):
+        model = GPT2LMHeadModel.from_pretrained('gpt2')
+        tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+        return model, tokenizer
+
+    def load_bias_model(self):
+        # Load the model for zero-shot classification
+        model = AutoModelForSequenceClassification.from_pretrained('Hate-speech-CNERG/dehatebert-mono-english')
+        tokenizer = AutoTokenizer.from_pretrained('Hate-speech-CNERG/dehatebert-mono-english')
+        
+        # Define label2id mapping for entailment and contradiction
+        model.config.label2id = {'not hate speech': 0, 'hate speech': 1}
+        
+        # Return pipeline with the proper model and tokenizer
+        return pipeline("zero-shot-classification", model=model, tokenizer=tokenizer)
+
+    def evaluate_bleu_rouge(self, candidates, references):
+        bleu_score = corpus_bleu(candidates, [references]).score
+        scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
+        rouge_scores = [scorer.score(ref, cand) for ref, cand in zip(references, candidates)]
+        rouge1 = sum([score['rouge1'].fmeasure for score in rouge_scores]) / len(rouge_scores)
+        return bleu_score, rouge1
+
+    def evaluate_bert_score(self, candidates, references):
+        P, R, F1 = score(candidates, references, lang="en", model_type='bert-base-multilingual-cased')
+        return P.mean().item(), R.mean().item(), F1.mean().item()
+
+    def evaluate_perplexity(self, text):
+        encodings = self.gpt2_tokenizer(text, return_tensors='pt')
+        max_length = self.gpt2_model.config.n_positions
+        stride = 512
+        lls = []
+        for i in range(0, encodings.input_ids.size(1), stride):
+            begin_loc = max(i + stride - max_length, 0)
+            end_loc = min(i + stride, encodings.input_ids.size(1))
+            trg_len = end_loc - i
+            input_ids = encodings.input_ids[:, begin_loc:end_loc]
+            target_ids = input_ids.clone()
+            target_ids[:, :-trg_len] = -100
+            with torch.no_grad():
+                outputs = self.gpt2_model(input_ids, labels=target_ids)
+                log_likelihood = outputs[0] * trg_len
+            lls.append(log_likelihood)
+        ppl = torch.exp(torch.stack(lls).sum() / end_loc)
+        return ppl.item()
+
+    def evaluate_diversity(self, texts):
+        all_tokens = [tok for text in texts for tok in text.split()]
+        unique_bigrams = set(ngrams(all_tokens, 2))
+        diversity_score = len(unique_bigrams) / len(all_tokens) if all_tokens else 0
+        return diversity_score
+
+    def evaluate_racial_bias(self, text):
+        results = self.bias_pipeline([text], candidate_labels=["hate speech", "not hate speech"])
+        bias_score = results[0]['scores'][results[0]['labels'].index('hate speech')]
+        return bias_score
+
+    def evaluate_meteor(self, candidates, references):
+        nltk.download('punkt', quiet=True)  
+        
+        meteor_scores = [
+            meteor_score([word_tokenize(ref)], word_tokenize(cand))
+            for ref, cand in zip(references, candidates)
+        ]
+        return sum(meteor_scores) / len(meteor_scores)
+    
+    def evaluate_chrf(self, candidates, references):
+        chrf_scores = [sentence_chrf(ref, cand) for ref, cand in zip(references, candidates)]
+        return sum(chrf_scores) / len(chrf_scores)
+    
+    def evaluate_readability(self, text):
+        flesch_ease = flesch_reading_ease(text)
+        flesch_grade = flesch_kincaid_grade(text)
+        return flesch_ease, flesch_grade
+
+    def evaluate_all(self, response, reference):
+        candidates = [response]
+        references = [reference]
+        bleu, rouge1 = self.evaluate_bleu_rouge(candidates, references)
+        bert_p, bert_r, bert_f1 = self.evaluate_bert_score(candidates, references)
+        perplexity = self.evaluate_perplexity(response)
+        diversity = self.evaluate_diversity(candidates)
+        racial_bias = self.evaluate_racial_bias(response)
+        meteor = self.evaluate_meteor(candidates, references)
+        chrf = self.evaluate_chrf(candidates, references)
+        flesch_ease, flesch_grade = self.evaluate_readability(response)
+        return {
+            "BLEU": bleu,
+            "ROUGE-1": rouge1,
+            "BERT P": bert_p,
+            "BERT R": bert_r,
+            "BERT F1": bert_f1,
+            "Perplexity": perplexity,
+            "Diversity": diversity,
+            "Racial Bias": racial_bias,
+            "METEOR": meteor,
+            "CHRF": chrf,
+            "Flesch Reading Ease": flesch_ease,
+            "Flesch-Kincaid Grade": flesch_grade,
+        }
+

memory.py

@@ -0,0 +1,147 @@
+import streamlit as st
+import openai
+from openai import OpenAI
+from brain import get_index_for_documents
+from langchain.chains import RetrievalQA
+from langchain_community.chat_models import ChatOpenAI
+from langchain_community.embeddings import OpenAIEmbeddings
+from langchain_community.vectorstores import FAISS
+from dotenv import load_dotenv
+import os
+from evaluation_module import RAGEvaluator
+
+# Set the title for the Streamlit app
+st.title("DocuChat with Evaluation")
+
+# Set up the OpenAI client
+client = OpenAI()
+load_dotenv()  # Load variables from .env
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+# Initialize evaluator
+evaluator = RAGEvaluator()
+
+# Function to create vector database from different file types
+@st.cache_resource
+def create_vectordb(files, filenames, raw_texts):
+    with st.spinner("Creating vector database..."):
+        vectordb = get_index_for_documents(
+            [file.getvalue() for file in files if file.type == "application/pdf"],
+            filenames,
+            [raw_text for raw_text in raw_texts.splitlines() if raw_text.strip()],
+            openai.api_key
+        )
+    return vectordb
+
+# Upload files using Streamlit's file uploader
+uploaded_files = st.file_uploader("Upload your documents (PDF or TXT)", type=["pdf", "txt"], accept_multiple_files=True, label_visibility="hidden")
+
+# Text area for raw text input
+raw_text = st.text_area("Or enter your raw text here:", height=150)
+
+# If files are uploaded or raw text is provided, create the vectordb and store it in the session state
+if uploaded_files or raw_text:
+    file_names = [file.name for file in uploaded_files] if uploaded_files else []
+    st.session_state["vectordb"] = create_vectordb(uploaded_files, file_names, raw_text)
+
+# Define the template for the chatbot prompt
+prompt_template = """
+    You are a helpful Assistant who answers to users questions based on multiple contexts given to you.
+
+    Keep your answer short and to the point.
+    
+    The evidence is the context of the document extract with metadata. 
+    
+    Carefully focus on the metadata, especially 'filename' and 'page' whenever answering.
+    
+    Make sure to add filename and page number at the end of the sentence you are citing to.
+
+    Also be able to give a summary based on the document extract given to you, but do not hallucinate.
+        
+    Reply "Not applicable" if text is irrelevant.
+     
+    The document content is:
+    {doc_extract}
+"""
+
+# Get the current prompt from the session state or set a default value
+prompt = st.session_state.get("prompt", [{"role": "system", "content": "none"}])
+
+# Display previous chat messages
+for message in prompt:
+    if message["role"] != "system":
+        with st.chat_message(message["role"]):
+            st.write(message["content"])
+
+# Get the user's question using Streamlit's chat input
+question = st.chat_input("Ask anything")
+
+# Handle the user's question
+if question:
+    vectordb = st.session_state.get("vectordb", None)
+    if not vectordb:
+        with st.chat_message("assistant"):
+            st.write("You need to provide a PDF, TXT file, or raw text.")
+            st.stop()
+
+    # Search the vectordb for similar content to the user's question
+    search_results = vectordb.similarity_search(question, k=3)
+    doc_extract = "\n".join([result.page_content for result in search_results])
+
+    # Update the prompt with the document extract
+    prompt[0] = {
+        "role": "system",
+        "content": prompt_template.format(doc_extract=doc_extract),
+    }
+
+    # Add the user's question to the prompt and display it
+    prompt.append({"role": "user", "content": question})
+    with st.chat_message("user"):
+        st.write(question)
+
+    # Display an empty assistant message while waiting for the response
+    with st.chat_message("assistant"):
+        botmsg = st.empty()
+
+    # Call ChatGPT with streaming and display the response as it comes
+    response = []
+    result = ""
+    for chunk in client.chat.completions.create(
+        model="gpt-3.5-turbo", messages=prompt, stream=True
+    ):
+        text = chunk.choices[0].delta.content
+        if text is not None:
+            response.append(text)
+            result = "".join(response).strip()
+            botmsg.write(result)
+
+    # Add the assistant's response to the prompt
+    prompt.append({"role": "assistant", "content": result})
+
+    # Store the updated prompt in the session state
+    st.session_state["prompt"] = prompt
+
+    # Evaluation Section
+    st.write("## Evaluation Results")
+    if st.button("Evaluate Response"):
+        if doc_extract and result:
+            # Perform evaluation
+            metrics = evaluator.evaluate_all(result, doc_extract)
+
+            # Display metrics with explanations
+            st.write(f"**BLEU Score**: {metrics['BLEU']:.2f}")
+            st.write("BLEU measures the overlap between the generated output and reference text based on n-grams. Range: 0-100. Higher scores indicate better match.")
+
+            st.write(f"**ROUGE-1 Score**: {metrics['ROUGE-1']:.2f}")
+            st.write("ROUGE-1 measures the overlap of unigrams between the generated output and reference text. Range: 0-1. Higher scores indicate better match.")
+
+            st.write(f"**BERT Precision**: {metrics['BERT P']:.2f}")
+            st.write(f"**BERT Recall**: {metrics['BERT R']:.2f}")
+            st.write(f"**BERT F1 Score**: {metrics['BERT F1']:.2f}")
+            st.write("BERTScore evaluates the semantic similarity between the generated output and reference text using BERT embeddings. Range: 0-1. Higher scores indicate better semantic similarity.")
+
+            st.write(f"**Perplexity**: {metrics['Perplexity']:.2f}")
+            st.write("Perplexity measures how well a language model predicts the text. Range: 1 to ∞. Lower values indicate better fluency and coherence.")
+
+            st.write(f"**Diversity**: {metrics['Diversity']:.2f}")
+            st.write("Diversity measures the uniqueness of bigrams in the generated output. Range: 0-1. Higher values indicate more diverse and varied output.")