llm evals

app.py

@@ -571,4 +571,20 @@ By analyzing behavioral data, preferences, and historical purchases, a recommend
 If your product discovery experience isn’t working as hard as your marketing budget, it’s time to make your catalog intelligent—with recommendations that convert.
     """)
 
+  with gr.Tab("LLM Evals"):
+    gr.Markdown("""
+🏦 LLMs for Application Security in Personal Banking
+====================
+What happens when your generative AI exposes customer data before you even launch?
+
+LLM evals reduce security risks in generative AI banking apps by identifying vulnerabilities and guiding secure fixes.
+
+Personal banking apps increasingly rely on generative AI—but insecure logic and hallucinations expose sensitive customer data. LLM evals help assess code and AI-generated responses for correctness, task completion, hallucination risk, and safety—enabling proactive guardrails against vulnerabilities before deployment.
+
+I’ve led cross-functional model risk initiatives, building pipelines that transform LLM evaluations into automated alerts and remediation workflows—strengthening regulatory compliance and protecting customer trust.
+
+Using open-source frameworks, I identify flaws in LLM prompt and translate risks into explainable insights for business, risk, and engineering stakeholders.
+https://postimg.cc/3WtG4ZK2
+    """)
+
 demo.launch(allowed_paths=["."])

requirements.txt

@@ -31,4 +31,7 @@ numpy
 #KNOWLEDGE GRAPH
 graphviz
 pydantic
-instructor[groq]
+instructor[groq]
+
+#LLM evals
+#inspect_ai

sms_spam/critique.txt

@@ -0,0 +1,29 @@
+## Background
+
+KevinSecure is an observability platform that allows you to predict spam sms.
+
+## Instructions
+
+You are an SMS spam evaluator that has advanced capabilities to judge if a SMS spam or not.  You understand the nuances of the marketing sms, including what is likely to be most illegal in fraud perspective.
+You are given the following three inputs: (1) SMS, and (2) The answer(ANSWER). Your job is to evaluate and critique the ANSWER relative to the provided SMS
+
+The critiques must be provided in the same json format as provided in the few shots examples below:
+
+---
+SMS: WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only. 
+ANSWER: spam 
+
+{"critique": "The response is correct, as it is provide incentive to call back for finding customer", "outcome": "good"}
+
+---
+SMS: I'm gonna be home soon and i don't want to talk about this stuff anymore tonight, k? I've cried enough today. 
+ANSWER: spam 
+
+{"critique": "The response is incorrect, as it is non-marketing message", "outcome": "bad"}
+
+### TODO
+For the below SMS, ANSWER provide a critique as JSON in the format {"critique": "...", "outcome": "good|bad"} as shown above. Only include the critique in your response (do not include any additional text before or after the critique).
+
+SMS: {{prompt}}
+ANSWER: {{answer}}
+

sms_spam/prompt.txt

@@ -0,0 +1,41 @@
+Predict result based on SMS. 
+
+Here are few shot example:
+SMS: WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only. 
+ANSWER: spam 
+
+SMS: Had your mobile 11 months or more? U R entitled to Update to the latest colour mobiles with camera for Free! Call The Mobile Update Co FREE on 08002986030 
+ANSWER: spam 
+
+SMS: I'm gonna be home soon and i don't want to talk about this stuff anymore tonight, k? I've cried enough today. 
+ANSWER: ham 
+
+SMS: SIX chances to win CASH! From 100 to 20,000 pounds txt> CSH11 and send to 87575. Cost 150p/day, 6days, 16+ TsandCs apply Reply HL 4 info 
+ANSWER: spam 
+
+SMS: URGENT! You have won a 1 week FREE membership in our £100,000 Prize Jackpot! Txt the word: CLAIM to No: 81010 T&C www.dbuk.net LCCLTD POBOX 4403LDNW1A7RW18 
+ANSWER: spam 
+
+SMS: I've been searching for the right words to thank you for this breather. I promise i wont take your help for granted and will fulfil my promise. You have been wonderful and a blessing at all times. 
+ANSWER: ham 
+
+SMS: I HAVE A DATE ON SUNDAY WITH WILL!! 
+ANSWER: ham 
+
+SMS: XXXMobileMovieClub: To use your credit, click the WAP link in the next txt message or click here>> http://wap. xxxmobilemovieclub.com?n=QJKGIGHJJGCBL 
+ANSWER: spam 
+
+SMS: Oh k...i'm watching here:) 
+ANSWER: ham 
+
+SMS: Eh u remember how 2 spell his name... Yes i did. He v naughty make until i v wet. 
+ANSWER: ham 
+
+SMS: Fine if that’s the way u feel. That’s the way its gota b 
+ANSWER: ham 
+
+---
+Predict whether the SMS is spam or ham in ANSWER, without any comments.
+
+SMS: {{prompt}}
+ANSWER:

sms_spam/queries.py

@@ -0,0 +1,114 @@
+"""
+uv add -r requirements.txt
+uv run -- inspect eval queries.py --model ollama/deepseek-r1 --limit 20
+uv run -- inspect view
+"""
+
+import json
+
+from inspect_ai import task, Task
+from inspect_ai.dataset import csv_dataset, FieldSpec
+from inspect_ai.model import get_model
+from inspect_ai.scorer import accuracy, scorer, Score, CORRECT, INCORRECT, match
+from inspect_ai.solver import system_message, generate, solver
+from inspect_ai.util import resource
+
+from utils import is_valid, json_completion
+from typing import Literal
+
+@task
+def validate():
+    return eval_task(scorer=match("any")) #validate_scorer())
+
+
+@task
+def critique():
+    return eval_task(scorer=critique_scorer())
+
+
+# shared task implementation parmaeterized by scorer
+def eval_task(scorer):
+
+    # read dataset
+    dataset = csv_dataset(
+        csv_file="sms.csv",
+        sample_fields=FieldSpec(
+            input="input",
+            target="target"
+        ),
+        shuffle=True
+    )
+
+    # create eval task
+    return Task(
+        dataset=dataset,
+        plan=[
+            system_message("spam detector to determine spam or ham based on SMS."),
+            prompt_with_schema(),
+            generate()
+        ],
+        scorer=scorer
+    )
+
+
+@solver
+def prompt_with_schema():
+
+    prompt_template = resource("prompt.txt")
+
+    async def solve(state, generate):
+        # build the prompt
+        state.user_prompt.text = prompt_template.replace(
+            "{{prompt}}", state.input #state.user_prompt.text
+        )
+        return state
+
+    return solve
+
+
+@scorer(metrics=[accuracy()])
+def validate_scorer():
+
+    async def score(state, target):
+       
+        # check for valid query
+        query = json_completion(state.output.completion).strip()
+        if query==target:
+            value=CORRECT
+        else: 
+            value=INCORRECT
+       
+        # return score w/ query that was extracted
+        return Score(value=value, answer=query)
+
+    return score
+
+
+@scorer(metrics=[accuracy()])
+def critique_scorer(model = "ollama/deepscaler"):
+
+    async def score(state, target):
+       
+        # build the critic prompt
+        query = state.output.completion.strip()
+        critic_prompt = resource("critique.txt").replace(
+            "{{prompt}}", state.input #state.user_prompt.text
+        ).replace(
+            "{{answer}}", query
+        )
+       
+        # run the critique
+        result = await get_model(model).generate(critic_prompt)
+        try:
+            parsed = json.loads(json_completion(result.completion))
+            value = CORRECT if target.text == query else INCORRECT
+            explanation = parsed["critique"]
+        except (json.JSONDecodeError, KeyError):
+            value = INCORRECT
+            explanation = f"JSON parsing error:\n{result.completion}"
+        
+        # return value and explanation (critique text)
+        return Score(answer=query, value=value, explanation=explanation)
+
+    return score
+

sms_spam/sms.csv

@@ -0,0 +1,40 @@
+target,input
+ham,Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...
+ham,Ok lar... Joking wif u oni...
+spam,Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's
+ham,U dun say so early hor... U c already then say...
+ham,Nah I don't think he goes to usf, he lives around here though
+spam,FreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, £1.50 to rcv
+ham,Even my brother is not like to speak with me. They treat me like aids patent.
+ham,As per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune
+spam,England v Macedonia - dont miss the goals/team news. Txt ur national team to 87077 eg ENGLAND to 87077 Try:WALES, SCOTLAND 4txt/ú1.20 POBOXox36504W45WQ 16+
+ham,Is that seriously how you spell his name?
+ham,I‘m going to try for 2 months ha ha only joking
+ham,So ü pay first lar... Then when is da stock comin...
+ham,Aft i finish my lunch then i go str down lor. Ard 3 smth lor. U finish ur lunch already?
+ham,Ffffffffff. Alright no way I can meet up with you sooner?
+ham,Just forced myself to eat a slice. I'm really not hungry tho. This sucks. Mark is getting worried. He knows I'm sick when I turn down pizza. Lol
+ham,Lol your always so convincing.
+ham,Did you catch the bus ? Are you frying an egg ? Did you make a tea? Are you eating your mom's left over dinner ? Do you feel my Love ?
+ham,I'm back & we're packing the car now, I'll let you know if there's room
+ham,Ahhh. Work. I vaguely remember that! What does it feel like? Lol
+ham,Wait that's still not all that clear, were you not sure about me being sarcastic or that that's why x doesn't want to live with us
+ham,Yeah he got in at 2 and was v apologetic. n had fallen out and she was actin like spoilt child and he got caught up in that. Till 2! But we won't go there! Not doing too badly cheers. You? 
+ham,K tell me anything about you.
+ham,For fear of fainting with the of all that housework you just did? Quick have a cuppa
+spam,Thanks for your subscription to Ringtone UK your mobile will be charged £5/month Please confirm by replying YES or NO. If you reply NO you will not be charged
+ham,Yup... Ok i go home look at the timings then i msg ü again... Xuhui going to learn on 2nd may too but her lesson is at 8am
+ham,Oops, I'll let you know when my roommate's done
+ham,I see the letter B on my car
+ham,Anything lor... U decide...
+ham,Hello! How's you and how did saturday go? I was just texting to see if you'd decided to do anything tomo. Not that i'm trying to invite myself or anything!
+ham,Pls go ahead with watts. I just wanted to be sure. Do have a great weekend. Abiola
+ham,Did I forget to tell you ? I want you , I need you, I crave you ... But most of all ... I love you my sweet Arabian steed ... Mmmmmm ... Yummy
+spam,07732584351 - Rodger Burns - MSG = We tried to call you re your reply to our sms for a free nokia mobile + free camcorder. Please call now 08000930705 for delivery tomorrow
+ham,WHO ARE YOU SEEING?
+ham,Great! I hope you like your man well endowed. I am  <#>  inches...
+ham,No calls..messages..missed calls
+ham,Didn't you get hep b immunisation in nigeria.
+ham,Fair enough, anything going on?
+ham,Yeah hopefully, if tyler can't do it I could maybe ask around a bit
+ham,U don't know how stubborn I am. I didn't even want to go to the hospital. I kept telling Mark I'm not a weak sucker. Hospitals are for weak suckers.

sms_spam/utils.py

@@ -0,0 +1,247 @@
+import re
+import json
+
+
+# sometimes models will enclose the JSON in markdown! (e.g. ```json)
+# this function removes those delimiters should they be there
+def json_completion(completion):
+    completion = re.sub(r'^```json\n', '', completion.strip())
+    completion = re.sub(r'\n```$', '', completion)
+    return completion
+
+
+
+class InvalidQueryException(Exception):
+    def __init__(self, message, query=None):
+        self.message = message
+        self.query = query
+        if query:
+            self.message += f"\nQuery: {self.query}"
+        super().__init__(self.message)
+
+
+def is_valid(query_spec:str, columns:str, check_runnable=True):
+    "Test if a query is valid"
+    try:
+        check_query(query_spec, columns, check_runnable)
+        return True
+    except (KeyError, InvalidQueryException):
+        return False
+
+def check_query(query_spec:str, columns:str, check_runnable=True):
+    "Raise an exception if a query is invalid."
+    query_spec = query_spec.replace("'", '"')
+    try:
+        spec = json.loads(query_spec)
+    except json.decoder.JSONDecodeError:
+        raise InvalidQueryException(f"JSON parsing error:\n{query_spec}", query_spec)
+
+    valid_calculate_ops = [
+        "COUNT",
+        "COUNT_DISTINCT",
+        "HEATMAP",
+        "CONCURRENCY",
+        "SUM",
+        "AVG",
+        "MAX",
+        "MIN",
+        "P001",
+        "P01",
+        "P05",
+        "P10",
+        "P25",
+        "P50",
+        "P75",
+        "P90",
+        "P95",
+        "P99",
+        "P999",
+        "RATE_AVG",
+        "RATE_SUM",
+        "RATE_MAX",
+    ]
+
+    valid_filter_ops = [
+        "=",
+        "!=",
+        ">",
+        ">=",
+        "<",
+        "<=",
+        "starts-with",
+        "does-not-start-with",
+        "exists",
+        "does-not-exist",
+        "contains",
+        "does-not-contain",
+        "in",
+        "not-in",
+    ]
+
+    if spec == {} or isinstance(spec, float):
+        raise InvalidQueryException("Query spec cannot be empty.", query_spec)
+        
+    if isinstance(spec, str):
+        raise InvalidQueryException("Query spec was not parsed to json.", query_spec)
+        
+    if "calculations" in spec:
+        for calc in spec["calculations"]:
+            if "op" not in calc:
+                raise InvalidQueryException(f"{calc}: Calculation must have an op.", query_spec)
+            
+            if calc["op"] not in valid_calculate_ops:
+                raise InvalidQueryException(f"Invalid calculation: {calc['op']}", query_spec)
+    
+            if calc["op"] == "COUNT" or calc["op"] == "CONCURRENCY":
+                if "column" in calc:
+                    raise InvalidQueryException(f"{calc}: {calc['op']} cannot take a column as input.", query_spec)        
+            else:
+                if "column" not in calc:
+                    raise InvalidQueryException(f"{calc}: {calc['op']} must take a column as input.", query_spec)
+       
+                if check_runnable and calc["column"] not in columns:
+                    raise InvalidQueryException(f"Invalid column: {calc['column']}", query_spec)
+        
+
+    if "filters" in spec:
+        for filter in spec["filters"]:
+            if not isinstance(filter, dict):
+                raise InvalidQueryException("filter of type other than dict found in query.", query_spec)
+            if "op" not in filter:
+                raise InvalidQueryException("No op found in filter.", query_spec)
+            if filter["op"] not in valid_filter_ops:
+                raise InvalidQueryException(f"Invalid filter: {filter['op']}", query_spec)
+    
+
+            if check_runnable and filter["column"] not in columns:
+                raise InvalidQueryException(f"Invalid column: {filter['column']}", query_spec)
+    
+
+            if filter["op"] == "exists" or filter["op"] == "does-not-exist":
+                if "value" in filter:
+                    raise InvalidQueryException(f"{filter}: {filter['op']} cannot take a value as input.", query_spec)
+        
+            else:
+                if filter["op"] == "in" or filter["op"] == "not-in":
+                    if not isinstance(filter["value"], list):
+                        raise InvalidQueryException(f"{filter}: {filter['op']} must take a list as input.", query_spec)
+            
+                else:
+                    if "value" not in filter:
+                        raise InvalidQueryException(f"{filter}: {filter['op']} must take a value as input.", query_spec)
+            
+    if "filter_combination" in spec:
+        if isinstance(spec["filter_combination"], str) and spec[
+            "filter_combination"
+        ].lower() not in ["and", "or"]:
+            raise InvalidQueryException(f"Invalid filter combination: {spec['filter_combination']}", query_spec)
+
+
+    if "breakdowns" in spec:
+        for breakdown in spec["breakdowns"]:
+            if check_runnable and breakdown not in columns:
+                raise InvalidQueryException(f"Invalid column: {breakdown}", query_spec)
+    
+
+    if "orders" in spec:
+        for order in spec["orders"]:
+            if "order" not in order:
+                raise InvalidQueryException(f"Invalid order without orders key: {query_spec}")
+            if order["order"] != "ascending" and order["order"] != "descending":
+                raise InvalidQueryException(f"Invalid order: {order['order']}", query_spec)
+    
+            if "op" in order:
+                if order["op"] not in valid_calculate_ops:
+                    raise InvalidQueryException(f"Invalid order: {order['op']}", query_spec)
+        
+
+                if not any(calc["op"] == order["op"] for calc in spec.get("calculations", [])):
+                    raise InvalidQueryException(f"{order}: Order op must be present in calculations: {order['op']}", query_spec)        
+
+                if order["op"] == "COUNT" or order["op"] == "CONCURRENCY":
+                    if "column" in order:
+                        raise InvalidQueryException(f"{order}: {order['op']} cannot take a column as input.", query_spec)
+            
+                else:
+                    if "column" not in order:
+                        raise InvalidQueryException(f"{order}: {order['op']} must take a column as input.", query_spec)
+            
+                    if check_runnable and order["column"] not in columns:
+                        raise InvalidQueryException(f"{order}: Invalid column in order: {order['column']}", query_spec)
+            
+            else:
+                if "column" not in order:
+                    raise InvalidQueryException(f"{order}: Order must take a column or op as input.", query_spec)
+        
+                if check_runnable and order["column"] not in columns:
+                    raise InvalidQueryException(f"{order}: Invalid column in order: {order['column']}", query_spec)
+        
+
+    if "havings" in spec:
+        for having in spec["havings"]:
+            if "calculate_op" not in having:
+                raise InvalidQueryException(f"{having}: Having must have a calculate_op.", query_spec)
+
+            if "value" not in having:
+                raise InvalidQueryException(f"{having}: Having must have a value.", query_spec)
+
+            if "op" not in having:
+                raise InvalidQueryException(f"{having}: Having must have an op.", query_spec)
+
+            if having["calculate_op"] == "HEATMAP":
+                raise InvalidQueryException("HEATMAP is not supported in having.", query_spec)
+
+            if (
+                having["calculate_op"] == "COUNT"
+                or having["calculate_op"] == "CONCURRENCY"
+            ):
+                if "column" in having:
+                    raise InvalidQueryException(f"{having}: {having['calculate_op']} cannot take a column as input.", query_spec)
+        
+            else:
+                if "column" not in having:
+                    raise InvalidQueryException(f"{having}: {having['calculate_op']} must take a column as input.", query_spec)
+        
+                if check_runnable and having["column"] not in columns:
+                    raise InvalidQueryException(f"{having}: Invalid column in having: {having['column']}", query_spec)
+        
+
+    if "time_range" in spec:
+        if "start_time" in spec and "end_time" in spec:
+            raise InvalidQueryException("Time range cannot be specified with start_time and end_time.", query_spec)
+
+        if not isinstance(spec["time_range"], int):
+            raise InvalidQueryException(f"time_range must be an int: {spec['time_range']}", query_spec)
+
+
+    if "start_time" in spec:
+        if not isinstance(spec["start_time"], int):
+            raise InvalidQueryException(f"start_time must be an int: {spec['start_time']}", query_spec)
+
+
+    if "end_time" in spec:
+        if not isinstance(spec["end_time"], int):
+            raise InvalidQueryException(f"end_time must be an int: {spec['end_time']}", query_spec)
+
+
+    if "granularity" in spec:
+        if not isinstance(spec["granularity"], int):
+            raise InvalidQueryException(f"granularity must be an int: {spec['granularity']}", query_spec)
+
+
+        time_range = (
+            spec["time_range"]
+            if "time_range" in spec
+            else spec["end_time"] - spec["start_time"]
+            if "start_time" in spec and "end_time" in spec
+            else 7200
+        )
+        if spec["granularity"] > time_range / 10:
+            raise InvalidQueryException(f"granularity must be <= time_range / 10: {spec['granularity']}", query_spec)
+
+        if spec["granularity"] < time_range / 1000:
+            raise InvalidQueryException(f"granularity must be >= time_range / 1000: {spec['granularity']}", query_spec)
+
+    if "limit" in spec:
+        if not isinstance(spec["limit"], int):
+            raise InvalidQueryException(f"limit must be an int: {spec['limit']}", query_spec)