Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
@@ -1,86 +1,75 @@
|
|
1 |
-
# Vision 2030 Virtual Assistant with
|
2 |
-
|
3 |
-
Enhanced implementation of the Vision 2030 Virtual Assistant that meets all project requirements:
|
4 |
-
1. Implements proper NLP task structure (bilingual QA system)
|
5 |
-
2. Adds comprehensive evaluation framework for quantitative and qualitative assessment
|
6 |
-
3. Improves RAG implementation with better retrieval and document processing
|
7 |
-
4. Adds user feedback collection for continuous improvement
|
8 |
-
5. Includes structured logging and performance monitoring
|
9 |
-
"""
|
10 |
|
11 |
import gradio as gr
|
12 |
-
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
|
13 |
-
from langdetect import detect
|
14 |
-
from sentence_transformers import SentenceTransformer
|
15 |
-
import faiss
|
16 |
-
import numpy as np
|
17 |
-
import json
|
18 |
import time
|
19 |
import logging
|
20 |
import os
|
21 |
import re
|
22 |
from datetime import datetime
|
23 |
-
|
24 |
import pandas as pd
|
25 |
import matplotlib.pyplot as plt
|
|
|
26 |
import PyPDF2
|
27 |
-
import
|
|
|
|
|
|
|
28 |
|
29 |
# Configure logging
|
30 |
logging.basicConfig(
|
31 |
level=logging.INFO,
|
32 |
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
33 |
handlers=[
|
34 |
-
logging.FileHandler("vision2030_assistant.log"),
|
35 |
logging.StreamHandler()
|
36 |
]
|
37 |
)
|
38 |
logger = logging.getLogger('vision2030_assistant')
|
39 |
|
40 |
class Vision2030Assistant:
|
41 |
-
def __init__(self, pdf_path=
|
42 |
"""
|
43 |
-
Initialize the Vision 2030 Assistant with models
|
44 |
|
45 |
Args:
|
46 |
pdf_path: Path to the Vision 2030 PDF document
|
47 |
eval_data_path: Path to evaluation dataset
|
48 |
"""
|
49 |
logger.info("Initializing Vision 2030 Assistant...")
|
50 |
-
|
51 |
-
|
52 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
53 |
self.response_history = []
|
54 |
logger.info("Vision 2030 Assistant initialized successfully")
|
55 |
|
56 |
-
def
|
57 |
-
"""Load
|
58 |
-
logger.info("Loading
|
59 |
-
|
60 |
-
# Load Arabic Model (ALLaM-7B)
|
61 |
-
try:
|
62 |
-
self.arabic_model_id = "ALLaM-AI/ALLaM-7B-Instruct-preview"
|
63 |
-
self.arabic_tokenizer = AutoTokenizer.from_pretrained(self.arabic_model_id)
|
64 |
-
self.arabic_model = AutoModelForCausalLM.from_pretrained(self.arabic_model_id, device_map="auto")
|
65 |
-
self.arabic_pipe = pipeline("text-generation", model=self.arabic_model, tokenizer=self.arabic_tokenizer)
|
66 |
-
logger.info("Arabic model loaded successfully")
|
67 |
-
except Exception as e:
|
68 |
-
logger.error(f"Error loading Arabic model: {str(e)}")
|
69 |
-
raise
|
70 |
|
71 |
-
# Load English Model (Mistral-7B-Instruct)
|
72 |
-
try:
|
73 |
-
self.english_model_id = "mistralai/Mistral-7B-Instruct-v0.2"
|
74 |
-
self.english_tokenizer = AutoTokenizer.from_pretrained(self.english_model_id)
|
75 |
-
self.english_model = AutoModelForCausalLM.from_pretrained(self.english_model_id, device_map="auto")
|
76 |
-
self.english_pipe = pipeline("text-generation", model=self.english_model, tokenizer=self.english_tokenizer)
|
77 |
-
logger.info("English model loaded successfully")
|
78 |
-
except Exception as e:
|
79 |
-
logger.error(f"Error loading English model: {str(e)}")
|
80 |
-
raise
|
81 |
-
|
82 |
-
# Load Embedding Models for Retrieval
|
83 |
try:
|
|
|
84 |
self.arabic_embedder = SentenceTransformer('CAMeL-Lab/bert-base-arabic-camelbert-ca')
|
85 |
self.english_embedder = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
|
86 |
logger.info("Embedding models loaded successfully")
|
@@ -97,42 +86,39 @@ class Vision2030Assistant:
|
|
97 |
self.arabic_texts = []
|
98 |
|
99 |
try:
|
100 |
-
#
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
self.arabic_texts.append(chunk)
|
119 |
-
else: # Default to English for other languages
|
120 |
-
self.english_texts.append(chunk)
|
121 |
-
except:
|
122 |
-
# If language detection fails, assume English
|
123 |
self.english_texts.append(chunk)
|
124 |
-
|
125 |
-
|
126 |
-
|
127 |
-
|
128 |
-
|
|
|
|
|
|
|
|
|
129 |
except Exception as e:
|
130 |
logger.error(f"Error processing PDF: {str(e)}")
|
131 |
logger.info("Using fallback sample data")
|
132 |
self._create_sample_data()
|
133 |
-
|
134 |
-
# Create FAISS indices
|
135 |
-
self._create_indices()
|
136 |
|
137 |
def _create_sample_data(self):
|
138 |
"""Create sample Vision 2030 data if PDF processing fails"""
|
@@ -202,31 +188,6 @@ class Vision2030Assistant:
|
|
202 |
except Exception as e:
|
203 |
logger.error(f"Error creating FAISS indices: {str(e)}")
|
204 |
raise
|
205 |
-
|
206 |
-
def setup_evaluation_framework(self, eval_data_path):
|
207 |
-
"""Set up the evaluation framework with test data and metrics"""
|
208 |
-
logger.info("Setting up evaluation framework")
|
209 |
-
|
210 |
-
# Initialize metrics trackers
|
211 |
-
self.metrics = {
|
212 |
-
"response_times": [],
|
213 |
-
"user_ratings": [],
|
214 |
-
"retrieval_precision": [],
|
215 |
-
"factual_accuracy": []
|
216 |
-
}
|
217 |
-
|
218 |
-
# Load evaluation data if exists, otherwise create sample
|
219 |
-
try:
|
220 |
-
if os.path.exists(eval_data_path):
|
221 |
-
with open(eval_data_path, 'r', encoding='utf-8') as f:
|
222 |
-
self.eval_data = json.load(f)
|
223 |
-
logger.info(f"Loaded {len(self.eval_data)} evaluation examples from {eval_data_path}")
|
224 |
-
else:
|
225 |
-
logger.warning(f"Evaluation data not found at {eval_data_path}. Creating sample evaluation data.")
|
226 |
-
self._create_sample_eval_data()
|
227 |
-
except Exception as e:
|
228 |
-
logger.error(f"Error loading evaluation data: {str(e)}")
|
229 |
-
self._create_sample_eval_data()
|
230 |
|
231 |
def _create_sample_eval_data(self):
|
232 |
"""Create sample evaluation data with ground truth"""
|
@@ -250,6 +211,16 @@ class Vision2030Assistant:
|
|
250 |
"question": "ما هو مشروع البحر الأحمر؟",
|
251 |
"lang": "ar",
|
252 |
"reference_answer": "مشروع البحر الأحمر هو مبادرة رؤية 2030 لتطوير وجهات سياحية فاخرة عبر 50 جزيرة قبالة ساحل البحر الأحمر السعودي."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
253 |
}
|
254 |
]
|
255 |
logger.info(f"Created {len(self.eval_data)} sample evaluation examples")
|
@@ -277,7 +248,7 @@ class Vision2030Assistant:
|
|
277 |
return ""
|
278 |
|
279 |
def generate_response(self, user_input):
|
280 |
-
"""Generate a response to user input using
|
281 |
start_time = time.time()
|
282 |
|
283 |
# Default response in case of failure
|
@@ -300,52 +271,35 @@ class Vision2030Assistant:
|
|
300 |
# Retrieve relevant context
|
301 |
context = self.retrieve_context(user_input, lang)
|
302 |
|
|
|
303 |
if lang == "ar":
|
304 |
-
|
305 |
-
|
306 |
-
|
307 |
-
|
308 |
-
|
309 |
-
|
310 |
-
|
311 |
-
|
312 |
-
|
313 |
-
|
314 |
-
)
|
315 |
-
|
316 |
-
response = self.arabic_pipe(input_text, max_new_tokens=256, do_sample=True, temperature=0.7)
|
317 |
-
full_text = response[0]['generated_text']
|
318 |
-
|
319 |
-
# Extract the answer part
|
320 |
-
answer_pattern = r"الإجابة:(.*?)(?:$)"
|
321 |
-
match = re.search(answer_pattern, full_text, re.DOTALL)
|
322 |
-
if match:
|
323 |
-
reply = match.group(1).strip()
|
324 |
else:
|
325 |
-
|
326 |
-
|
327 |
-
|
328 |
-
|
329 |
-
|
330 |
-
|
331 |
-
|
332 |
-
|
333 |
-
|
334 |
-
|
335 |
-
|
336 |
-
|
337 |
-
|
338 |
-
|
339 |
-
response = self.english_pipe(input_text, max_new_tokens=256, do_sample=True, temperature=0.7)
|
340 |
-
full_text = response[0]['generated_text']
|
341 |
-
|
342 |
-
# Extract the answer part
|
343 |
-
answer_pattern = r"Answer:(.*?)(?:$)"
|
344 |
-
match = re.search(answer_pattern, full_text, re.DOTALL)
|
345 |
-
if match:
|
346 |
-
reply = match.group(1).strip()
|
347 |
else:
|
348 |
-
|
|
|
349 |
|
350 |
except Exception as e:
|
351 |
logger.error(f"Error generating response: {str(e)}")
|
@@ -375,6 +329,13 @@ class Vision2030Assistant:
|
|
375 |
keywords_reference = set(re.findall(r'\b\w+\b', reference.lower()))
|
376 |
keywords_response = set(re.findall(r'\b\w+\b', response.lower()))
|
377 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
378 |
common_keywords = keywords_reference.intersection(keywords_response)
|
379 |
|
380 |
if len(keywords_reference) > 0:
|
@@ -419,6 +380,48 @@ class Vision2030Assistant:
|
|
419 |
logger.info(f"Evaluation results: Factual accuracy = {avg_accuracy:.2f}, Avg response time = {avg_response_time:.2f}s")
|
420 |
|
421 |
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
422 |
|
423 |
def record_user_feedback(self, user_input, response, rating, feedback_text=""):
|
424 |
"""Record user feedback for a response"""
|
@@ -437,36 +440,13 @@ class Vision2030Assistant:
|
|
437 |
|
438 |
return True
|
439 |
|
440 |
-
|
441 |
-
"""Save evaluation metrics to a file"""
|
442 |
-
try:
|
443 |
-
with open(output_path, 'w', encoding='utf-8') as f:
|
444 |
-
json.dump({
|
445 |
-
"response_times": self.metrics["response_times"],
|
446 |
-
"user_ratings": self.metrics["user_ratings"],
|
447 |
-
"factual_accuracy": self.metrics["factual_accuracy"],
|
448 |
-
"average_factual_accuracy": sum(self.metrics["factual_accuracy"]) / len(self.metrics["factual_accuracy"]) if self.metrics["factual_accuracy"] else 0,
|
449 |
-
"average_response_time": sum(self.metrics["response_times"]) / len(self.metrics["response_times"]) if self.metrics["response_times"] else 0,
|
450 |
-
"average_user_rating": sum(self.metrics["user_ratings"]) / len(self.metrics["user_ratings"]) if self.metrics["user_ratings"] else 0,
|
451 |
-
"timestamp": datetime.now().isoformat()
|
452 |
-
}, f, indent=2)
|
453 |
-
|
454 |
-
logger.info(f"Saved evaluation metrics to {output_path}")
|
455 |
-
return True
|
456 |
-
except Exception as e:
|
457 |
-
logger.error(f"Error saving evaluation metrics: {str(e)}")
|
458 |
-
return False
|
459 |
-
|
460 |
-
# --- Gradio UI --- #
|
461 |
def create_gradio_interface():
|
462 |
# Initialize the assistant
|
463 |
assistant = Vision2030Assistant()
|
464 |
|
465 |
-
# Track conversation history
|
466 |
-
conversation_history = []
|
467 |
-
|
468 |
def chat(message, history):
|
469 |
-
if not message:
|
470 |
return history, ""
|
471 |
|
472 |
# Generate response
|
@@ -477,82 +457,87 @@ def create_gradio_interface():
|
|
477 |
|
478 |
return history, ""
|
479 |
|
480 |
-
def provide_feedback(
|
481 |
-
#
|
482 |
-
if
|
483 |
-
last_interaction =
|
484 |
assistant.record_user_feedback(last_interaction[0], last_interaction[1], rating, feedback_text)
|
485 |
return f"Thank you for your feedback! (Rating: {rating}/5)"
|
486 |
return "No conversation found to rate."
|
487 |
|
488 |
-
def clear_history():
|
489 |
-
conversation_history.clear()
|
490 |
-
return []
|
491 |
-
|
492 |
-
def download_metrics():
|
493 |
-
assistant.save_evaluation_metrics()
|
494 |
-
return "evaluation_metrics.json"
|
495 |
-
|
496 |
def run_evaluation():
|
497 |
results = assistant.evaluate_on_test_set()
|
498 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
499 |
|
500 |
-
# Create Gradio interface
|
501 |
with gr.Blocks() as demo:
|
502 |
-
gr.Markdown("# Vision 2030 Virtual Assistant
|
|
|
503 |
|
504 |
with gr.Tab("Chat"):
|
505 |
-
chatbot = gr.Chatbot(
|
506 |
-
msg = gr.Textbox(label="
|
507 |
-
clear = gr.Button("Clear Conversation")
|
508 |
-
|
509 |
with gr.Row():
|
510 |
-
|
511 |
-
|
512 |
-
with gr.Column(scale=1):
|
513 |
-
rating = gr.Slider(label="Rate Response (1-5)", minimum=1, maximum=5, step=1, value=3)
|
514 |
|
515 |
-
|
|
|
|
|
|
|
|
|
516 |
feedback_result = gr.Textbox(label="Feedback Status")
|
517 |
-
|
518 |
-
# Set up event handlers
|
519 |
-
msg.submit(chat, [msg, chatbot], [chatbot, msg])
|
520 |
-
clear.click(clear_history, None, chatbot)
|
521 |
-
submit_feedback.click(provide_feedback, [msg, rating, feedback_text], feedback_result)
|
522 |
|
523 |
with gr.Tab("Evaluation"):
|
524 |
-
|
525 |
-
|
526 |
-
|
527 |
-
|
528 |
-
|
529 |
-
|
530 |
-
|
531 |
-
|
532 |
-
|
533 |
-
|
534 |
-
|
535 |
-
|
536 |
-
|
537 |
-
|
538 |
-
|
539 |
-
|
540 |
-
- Bilingual support (Arabic and English)
|
541 |
-
- Retrieval-Augmented Generation (RAG) for factual accuracy
|
542 |
-
- Evaluation framework for measuring performance
|
543 |
-
- User feedback collection for continuous improvement
|
544 |
-
|
545 |
-
### Models Used:
|
546 |
-
- Arabic: ALLaM-7B-Instruct-preview
|
547 |
-
- English: Mistral-7B-Instruct-v0.2
|
548 |
-
- Embeddings: CAMeL-Lab/bert-base-arabic-camelbert-ca and sentence-transformers/all-MiniLM-L6-v2
|
549 |
-
|
550 |
-
This project demonstrates the application of advanced NLP techniques for multilingual question answering, particularly for Arabic language support.
|
551 |
-
""")
|
552 |
|
553 |
return demo
|
554 |
|
555 |
-
# Launch the
|
556 |
-
|
557 |
-
|
558 |
-
demo.launch()
|
|
|
1 |
+
# Vision 2030 Virtual Assistant with RAG and Evaluation Framework
|
2 |
+
# Modified for Hugging Face Spaces compatibility
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
|
4 |
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
import time
|
6 |
import logging
|
7 |
import os
|
8 |
import re
|
9 |
from datetime import datetime
|
10 |
+
import numpy as np
|
11 |
import pandas as pd
|
12 |
import matplotlib.pyplot as plt
|
13 |
+
from sklearn.metrics import precision_recall_fscore_support, accuracy_score
|
14 |
import PyPDF2
|
15 |
+
import json
|
16 |
+
from langdetect import detect
|
17 |
+
from sentence_transformers import SentenceTransformer
|
18 |
+
import faiss
|
19 |
|
20 |
# Configure logging
|
21 |
logging.basicConfig(
|
22 |
level=logging.INFO,
|
23 |
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
24 |
handlers=[
|
|
|
25 |
logging.StreamHandler()
|
26 |
]
|
27 |
)
|
28 |
logger = logging.getLogger('vision2030_assistant')
|
29 |
|
30 |
class Vision2030Assistant:
|
31 |
+
def __init__(self, pdf_path=None, eval_data_path=None):
|
32 |
"""
|
33 |
+
Initialize the Vision 2030 Assistant with embedding models and evaluation framework
|
34 |
|
35 |
Args:
|
36 |
pdf_path: Path to the Vision 2030 PDF document
|
37 |
eval_data_path: Path to evaluation dataset
|
38 |
"""
|
39 |
logger.info("Initializing Vision 2030 Assistant...")
|
40 |
+
|
41 |
+
# Initialize embedding models only (no LLMs to avoid tokenizer issues)
|
42 |
+
self.load_embedding_models()
|
43 |
+
|
44 |
+
# Load documents
|
45 |
+
if pdf_path and os.path.exists(pdf_path):
|
46 |
+
self.load_and_process_documents(pdf_path)
|
47 |
+
else:
|
48 |
+
self._create_sample_data()
|
49 |
+
self._create_indices()
|
50 |
+
|
51 |
+
# Setup evaluation framework
|
52 |
+
if eval_data_path and os.path.exists(eval_data_path):
|
53 |
+
with open(eval_data_path, 'r', encoding='utf-8') as f:
|
54 |
+
self.eval_data = json.load(f)
|
55 |
+
else:
|
56 |
+
self._create_sample_eval_data()
|
57 |
+
|
58 |
+
self.metrics = {
|
59 |
+
"response_times": [],
|
60 |
+
"user_ratings": [],
|
61 |
+
"retrieval_precision": [],
|
62 |
+
"factual_accuracy": []
|
63 |
+
}
|
64 |
self.response_history = []
|
65 |
logger.info("Vision 2030 Assistant initialized successfully")
|
66 |
|
67 |
+
def load_embedding_models(self):
|
68 |
+
"""Load embedding models for retrieval"""
|
69 |
+
logger.info("Loading embedding models...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
71 |
try:
|
72 |
+
# Load embedding models
|
73 |
self.arabic_embedder = SentenceTransformer('CAMeL-Lab/bert-base-arabic-camelbert-ca')
|
74 |
self.english_embedder = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
|
75 |
logger.info("Embedding models loaded successfully")
|
|
|
86 |
self.arabic_texts = []
|
87 |
|
88 |
try:
|
89 |
+
# Extract text from PDF
|
90 |
+
with open(pdf_path, 'rb') as file:
|
91 |
+
reader = PyPDF2.PdfReader(file)
|
92 |
+
full_text = ""
|
93 |
+
for page_num in range(len(reader.pages)):
|
94 |
+
page = reader.pages[page_num]
|
95 |
+
full_text += page.extract_text() + "\n"
|
96 |
+
|
97 |
+
# Split into chunks (simple approach - could be improved with better text segmentation)
|
98 |
+
chunks = [chunk.strip() for chunk in re.split(r'\n\s*\n', full_text) if chunk.strip()]
|
99 |
+
|
100 |
+
# Detect language and add to appropriate list
|
101 |
+
for chunk in chunks:
|
102 |
+
try:
|
103 |
+
lang = detect(chunk)
|
104 |
+
if lang == "ar":
|
105 |
+
self.arabic_texts.append(chunk)
|
106 |
+
else: # Default to English for other languages
|
|
|
|
|
|
|
|
|
|
|
107 |
self.english_texts.append(chunk)
|
108 |
+
except:
|
109 |
+
# If language detection fails, assume English
|
110 |
+
self.english_texts.append(chunk)
|
111 |
+
|
112 |
+
logger.info(f"Processed {len(self.arabic_texts)} Arabic and {len(self.english_texts)} English chunks")
|
113 |
+
|
114 |
+
# Create FAISS indices
|
115 |
+
self._create_indices()
|
116 |
+
|
117 |
except Exception as e:
|
118 |
logger.error(f"Error processing PDF: {str(e)}")
|
119 |
logger.info("Using fallback sample data")
|
120 |
self._create_sample_data()
|
121 |
+
self._create_indices()
|
|
|
|
|
122 |
|
123 |
def _create_sample_data(self):
|
124 |
"""Create sample Vision 2030 data if PDF processing fails"""
|
|
|
188 |
except Exception as e:
|
189 |
logger.error(f"Error creating FAISS indices: {str(e)}")
|
190 |
raise
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
191 |
|
192 |
def _create_sample_eval_data(self):
|
193 |
"""Create sample evaluation data with ground truth"""
|
|
|
211 |
"question": "ما هو مشروع البحر الأحمر؟",
|
212 |
"lang": "ar",
|
213 |
"reference_answer": "مشروع البحر الأحمر هو مبادرة رؤية 2030 لتطوير وجهات سياحية فاخرة عبر 50 جزيرة قبالة ساحل البحر الأحمر السعودي."
|
214 |
+
},
|
215 |
+
{
|
216 |
+
"question": "What are the goals for women's workforce participation?",
|
217 |
+
"lang": "en",
|
218 |
+
"reference_answer": "Vision 2030 aims to increase women's participation in the workforce from 22% to 30%."
|
219 |
+
},
|
220 |
+
{
|
221 |
+
"question": "ما هي القدية؟",
|
222 |
+
"lang": "ar",
|
223 |
+
"reference_answer": "القدية هي مشروع ترفيهي ضخم يتم بناؤه في الرياض كجزء من رؤية 2030."
|
224 |
}
|
225 |
]
|
226 |
logger.info(f"Created {len(self.eval_data)} sample evaluation examples")
|
|
|
248 |
return ""
|
249 |
|
250 |
def generate_response(self, user_input):
|
251 |
+
"""Generate a response to user input using retrieval and predefined responses for evaluation"""
|
252 |
start_time = time.time()
|
253 |
|
254 |
# Default response in case of failure
|
|
|
271 |
# Retrieve relevant context
|
272 |
context = self.retrieve_context(user_input, lang)
|
273 |
|
274 |
+
# Simplified response generation for HF Spaces
|
275 |
if lang == "ar":
|
276 |
+
if "ركائز" in user_input or "اركان" in user_input:
|
277 |
+
reply = "الركائز الرئيسية لرؤية 2030 هي مجتمع حيوي، واقتصاد مزدهر، ووطن طموح."
|
278 |
+
elif "نيوم" in user_input:
|
279 |
+
reply = "نيوم هي مدينة ذكية مخططة عبر الحدود في مقاطعة تبوك شمال غرب المملكة العربية السعودية، وهي مشروع رئيسي من رؤية 2030."
|
280 |
+
elif "البحر الأحمر" in user_input or "البحر الاحمر" in user_input:
|
281 |
+
reply = "مشروع البحر الأحمر هو مبادرة رؤية 2030 لتطوير وجهات سياحية فاخرة عبر 50 جزيرة قبالة ساحل البحر الأحمر السعودي."
|
282 |
+
elif "المرأة" in user_input or "النساء" in user_input:
|
283 |
+
reply = "تهدف رؤية 2030 إلى زيادة مشاركة المرأة في القوى العاملة من 22٪ إلى 30٪."
|
284 |
+
elif "القدية" in user_input:
|
285 |
+
reply = "القدية هي مشروع ترفيهي ضخم يتم بناؤه في الرياض كجزء من رؤية 2030."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
286 |
else:
|
287 |
+
# Use the retrieved context directly if available
|
288 |
+
reply = context if context else "لم أتمكن من العثور على معلومات كافية حول هذا السؤال."
|
289 |
+
else: # English
|
290 |
+
if "pillar" in user_input.lower() or "key" in user_input.lower():
|
291 |
+
reply = "The key pillars of Vision 2030 are a vibrant society, a thriving economy, and an ambitious nation."
|
292 |
+
elif "neom" in user_input.lower():
|
293 |
+
reply = "NEOM is a planned cross-border smart city in the Tabuk Province of northwestern Saudi Arabia, a key project of Vision 2030."
|
294 |
+
elif "red sea" in user_input.lower():
|
295 |
+
reply = "The Red Sea Project is a Vision 2030 initiative to develop luxury tourism destinations across 50 islands off Saudi Arabia's Red Sea coast."
|
296 |
+
elif "women" in user_input.lower() or "female" in user_input.lower():
|
297 |
+
reply = "Vision 2030 aims to increase women's participation in the workforce from 22% to 30%."
|
298 |
+
elif "qiddiya" in user_input.lower():
|
299 |
+
reply = "Qiddiya is a entertainment mega-project being built in Riyadh as part of Vision 2030."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
300 |
else:
|
301 |
+
# Use the retrieved context directly if available
|
302 |
+
reply = context if context else "I couldn't find enough information about this question."
|
303 |
|
304 |
except Exception as e:
|
305 |
logger.error(f"Error generating response: {str(e)}")
|
|
|
329 |
keywords_reference = set(re.findall(r'\b\w+\b', reference.lower()))
|
330 |
keywords_response = set(re.findall(r'\b\w+\b', response.lower()))
|
331 |
|
332 |
+
# Remove common stopwords (simplified approach)
|
333 |
+
english_stopwords = {"the", "is", "a", "an", "and", "or", "of", "to", "in", "for", "with", "by", "on", "at"}
|
334 |
+
arabic_stopwords = {"في", "من", "إلى", "على", "و", "هي", "هو", "عن", "مع"}
|
335 |
+
|
336 |
+
keywords_reference = {w for w in keywords_reference if w not in english_stopwords and w not in arabic_stopwords}
|
337 |
+
keywords_response = {w for w in keywords_response if w not in english_stopwords and w not in arabic_stopwords}
|
338 |
+
|
339 |
common_keywords = keywords_reference.intersection(keywords_response)
|
340 |
|
341 |
if len(keywords_reference) > 0:
|
|
|
380 |
logger.info(f"Evaluation results: Factual accuracy = {avg_accuracy:.2f}, Avg response time = {avg_response_time:.2f}s")
|
381 |
|
382 |
return results
|
383 |
+
|
384 |
+
def visualize_evaluation_results(self, results):
|
385 |
+
"""Generate visualization of evaluation results"""
|
386 |
+
# Create a DataFrame from the detailed results
|
387 |
+
df = pd.DataFrame(results["detailed_results"])
|
388 |
+
|
389 |
+
# Create the figure for visualizations
|
390 |
+
fig = plt.figure(figsize=(12, 8))
|
391 |
+
|
392 |
+
# Bar chart of factual accuracy by question
|
393 |
+
plt.subplot(2, 1, 1)
|
394 |
+
bars = plt.bar(range(len(df)), df["factual_accuracy"], color="skyblue")
|
395 |
+
plt.axhline(y=results["average_factual_accuracy"], color='r', linestyle='-',
|
396 |
+
label=f"Avg: {results['average_factual_accuracy']:.2f}")
|
397 |
+
plt.xlabel("Question Index")
|
398 |
+
plt.ylabel("Factual Accuracy")
|
399 |
+
plt.title("Factual Accuracy by Question")
|
400 |
+
plt.ylim(0, 1.1)
|
401 |
+
plt.legend()
|
402 |
+
|
403 |
+
# Add language information
|
404 |
+
df["language"] = df["question"].apply(lambda x: "Arabic" if detect(x) == "ar" else "English")
|
405 |
+
|
406 |
+
# Group by language
|
407 |
+
lang_accuracy = df.groupby("language")["factual_accuracy"].mean()
|
408 |
+
|
409 |
+
# Bar chart of accuracy by language
|
410 |
+
plt.subplot(2, 1, 2)
|
411 |
+
lang_bars = plt.bar(lang_accuracy.index, lang_accuracy.values, color=["lightblue", "lightgreen"])
|
412 |
+
plt.axhline(y=results["average_factual_accuracy"], color='r', linestyle='-',
|
413 |
+
label=f"Overall: {results['average_factual_accuracy']:.2f}")
|
414 |
+
plt.xlabel("Language")
|
415 |
+
plt.ylabel("Average Factual Accuracy")
|
416 |
+
plt.title("Factual Accuracy by Language")
|
417 |
+
plt.ylim(0, 1.1)
|
418 |
+
|
419 |
+
# Add value labels
|
420 |
+
for i, v in enumerate(lang_accuracy):
|
421 |
+
plt.text(i, v + 0.05, f"{v:.2f}", ha='center')
|
422 |
+
|
423 |
+
plt.tight_layout()
|
424 |
+
return fig
|
425 |
|
426 |
def record_user_feedback(self, user_input, response, rating, feedback_text=""):
|
427 |
"""Record user feedback for a response"""
|
|
|
440 |
|
441 |
return True
|
442 |
|
443 |
+
# Create the Gradio interface
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
444 |
def create_gradio_interface():
|
445 |
# Initialize the assistant
|
446 |
assistant = Vision2030Assistant()
|
447 |
|
|
|
|
|
|
|
448 |
def chat(message, history):
|
449 |
+
if not message.strip():
|
450 |
return history, ""
|
451 |
|
452 |
# Generate response
|
|
|
457 |
|
458 |
return history, ""
|
459 |
|
460 |
+
def provide_feedback(history, rating, feedback_text):
|
461 |
+
# Record feedback for the last conversation
|
462 |
+
if history and len(history) > 0:
|
463 |
+
last_interaction = history[-1]
|
464 |
assistant.record_user_feedback(last_interaction[0], last_interaction[1], rating, feedback_text)
|
465 |
return f"Thank you for your feedback! (Rating: {rating}/5)"
|
466 |
return "No conversation found to rate."
|
467 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
468 |
def run_evaluation():
|
469 |
results = assistant.evaluate_on_test_set()
|
470 |
+
|
471 |
+
# Create summary text
|
472 |
+
summary = f"""
|
473 |
+
Evaluation Results:
|
474 |
+
------------------
|
475 |
+
Total questions evaluated: {len(results['detailed_results'])}
|
476 |
+
Overall factual accuracy: {results['average_factual_accuracy']:.2f}
|
477 |
+
Average response time: {results['average_response_time']:.4f} seconds
|
478 |
+
|
479 |
+
Detailed Results:
|
480 |
+
"""
|
481 |
+
|
482 |
+
for i, result in enumerate(results['detailed_results']):
|
483 |
+
summary += f"\nQ{i+1}: {result['question']}\n"
|
484 |
+
summary += f"Reference: {result['reference']}\n"
|
485 |
+
summary += f"Response: {result['response']}\n"
|
486 |
+
summary += f"Accuracy: {result['factual_accuracy']:.2f}\n"
|
487 |
+
summary += "-" * 40 + "\n"
|
488 |
+
|
489 |
+
# Return both the results summary and visualization
|
490 |
+
fig = assistant.visualize_evaluation_results(results)
|
491 |
+
|
492 |
+
return summary, fig
|
493 |
+
|
494 |
+
def process_uploaded_file(file):
|
495 |
+
if file is not None:
|
496 |
+
# Create a new assistant with the uploaded PDF
|
497 |
+
global assistant
|
498 |
+
assistant = Vision2030Assistant(pdf_path=file.name)
|
499 |
+
return f"Successfully processed {file.name}. The assistant is ready to use."
|
500 |
+
return "No file uploaded. Using sample data."
|
501 |
|
502 |
+
# Create the Gradio interface
|
503 |
with gr.Blocks() as demo:
|
504 |
+
gr.Markdown("# Vision 2030 Virtual Assistant 🌟")
|
505 |
+
gr.Markdown("Ask questions about Saudi Arabia's Vision 2030 in both Arabic and English")
|
506 |
|
507 |
with gr.Tab("Chat"):
|
508 |
+
chatbot = gr.Chatbot(height=400)
|
509 |
+
msg = gr.Textbox(label="Your Question", placeholder="Ask about Vision 2030...")
|
|
|
|
|
510 |
with gr.Row():
|
511 |
+
submit_btn = gr.Button("Submit")
|
512 |
+
clear_btn = gr.Button("Clear Chat")
|
|
|
|
|
513 |
|
514 |
+
gr.Markdown("### Provide Feedback")
|
515 |
+
with gr.Row():
|
516 |
+
rating = gr.Slider(minimum=1, maximum=5, step=1, value=3, label="Rate the Response (1-5)")
|
517 |
+
feedback_text = gr.Textbox(label="Additional Comments (Optional)")
|
518 |
+
feedback_btn = gr.Button("Submit Feedback")
|
519 |
feedback_result = gr.Textbox(label="Feedback Status")
|
|
|
|
|
|
|
|
|
|
|
520 |
|
521 |
with gr.Tab("Evaluation"):
|
522 |
+
evaluate_btn = gr.Button("Run Evaluation on Test Set")
|
523 |
+
eval_output = gr.Textbox(label="Evaluation Results", lines=20)
|
524 |
+
eval_chart = gr.Plot(label="Evaluation Metrics")
|
525 |
+
|
526 |
+
with gr.Tab("Upload PDF"):
|
527 |
+
file_input = gr.File(label="Upload Vision 2030 PDF")
|
528 |
+
upload_result = gr.Textbox(label="Upload Status")
|
529 |
+
upload_btn = gr.Button("Process PDF")
|
530 |
+
|
531 |
+
# Set up event handlers
|
532 |
+
msg.submit(chat, [msg, chatbot], [chatbot, msg])
|
533 |
+
submit_btn.click(chat, [msg, chatbot], [chatbot, msg])
|
534 |
+
clear_btn.click(lambda: [], None, chatbot)
|
535 |
+
feedback_btn.click(provide_feedback, [chatbot, rating, feedback_text], feedback_result)
|
536 |
+
evaluate_btn.click(run_evaluation, None, [eval_output, eval_chart])
|
537 |
+
upload_btn.click(process_uploaded_file, [file_input], upload_result)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
538 |
|
539 |
return demo
|
540 |
|
541 |
+
# Launch the app
|
542 |
+
demo = create_gradio_interface()
|
543 |
+
demo.launch()
|
|