Update app.py

Fallback to Gradio

app.py

@@ -1,125 +1,71 @@
-import streamlit as st
+import gradio as gr
 import os
 
-from typing import Iterator
 from huggingface_hub import InferenceClient
 
+"""
+For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+"""
+
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
 
-# Configure page settings
-st.set_page_config(
-    page_title="LLM Taiwan Chat",
-    page_icon="💬",
-    layout="centered"
-)
 
-# Initialize session state for chat history and system prompt
-if "messages" not in st.session_state:
-    st.session_state.messages = []
-if "system_prompt" not in st.session_state:
-    st.session_state.system_prompt = "你是一個產自台灣的聊天機械人, 你以台灣本地人的身份, 使用正體中文回答問題."
-if "temperature" not in st.session_state:
-    st.session_state.temperature = 0.2
-if "top_p" not in st.session_state:
-    st.session_state.top_p = 0.95
-
-## model="lianghsun/Llama-3.2-Taiwan-3B" to meta-llama/Llama-3.2-3B-Instruct
-
-def stream_chat(prompt: str) -> Iterator[str]:
-    """Stream chat responses from the LLM API"""
-
-    client = InferenceClient(model="meta-llama/Llama-3.2-3B-Instruct", timeout=30, token=HF_TOKEN)
-
-    
-    messages = []
-    if st.session_state.system_prompt:
-        messages.append({"role": "system", "content": st.session_state.system_prompt})
-    messages.extend(st.session_state.messages)
-    
-    stream = client.chat.completions.create(
-        messages=messages,
-        model="meta-llama/Llama-3.2-3B-Instruct",
+### Model used changes from HuggingFaceH4/zephyr-7b-beta to meta-llama/Llama-3.2-3B-Instruct
+client = InferenceClient("lianghsun/Llama-3.2-Taiwan-3B", timeout=30, token=HF_TOKEN)
+
+
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    messages = [{"role": "system", "content": system_message}]
+
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
+
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
         stream=True,
-        temperature=st.session_state.temperature,
-        top_p=st.session_state.top_p
-    )
-    
-    for chunk in stream:
-        if chunk.choices[0].delta.content is not None:
-            yield chunk.choices[0].delta.content
-
-def clear_chat_history():
-    """Clear all chat messages and reset system prompt"""
-    st.session_state.messages = []
-    st.session_state.system_prompt = ""
-
-def main():
-    st.title("💬 LLM Taiwan Chat")
-    
-    # Add a clear chat button with custom styling
-    col1, col2 = st.columns([6, 1])
-    with col2:
-        if st.button("🗑️", type="secondary", use_container_width=True):
-            clear_chat_history()
-            st.rerun()
-    
-    # Advanced options in expander
-    with st.expander("進階選項 ⚙️", expanded=False):
-        # System prompt input
-        system_prompt = st.text_area(
-            "System Prompt 設定:",
-            value=st.session_state.system_prompt,
-            help="設定 system prompt 來定義 AI 助理的行為和角色。開始對話後將無法修改。",
-            height=100,
-            disabled=len(st.session_state.messages) > 0  # 當有對話時設為唯讀
-        )
-        if not st.session_state.messages and system_prompt != st.session_state.system_prompt:
-            st.session_state.system_prompt = system_prompt
-        
-        st.session_state.temperature = st.slider(
-            "Temperature",
-            min_value=0.0,
-            max_value=2.0,
-            value=st.session_state.temperature,
-            step=0.1,
-            help="較高的值會使輸出更加隨機，較低的值會使其更加集中和確定。"
-        )
-        st.session_state.top_p = st.slider(
-            "Top P",
-            min_value=0.1,
-            max_value=1.0,
-            value=st.session_state.top_p,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = message.choices[0].delta.content
+
+        response += token
+        yield response
+
+
+"""
+For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
+"""
+demo = gr.ChatInterface(
+    respond,
+    additional_inputs=[
+        gr.Textbox(value="你是一個產自台灣的聊天機械人, 你以台灣本地人的身份, 使用正體中文回答問題.", label="System message"),
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
             step=0.05,
-            help="控制模型輸出的多樣性，較低的值會使輸出更加保守。"
-        )
-    
-    # Display chat messages
-    for message in st.session_state.messages:
-        with st.chat_message(message["role"]):
-            st.write(message["content"])
-    
-    # Chat input
-    if prompt := st.chat_input("輸入您的訊息..."):
-        # Add user message to chat history
-        st.session_state.messages.append({"role": "user", "content": prompt})
-        
-        # Display user message
-        with st.chat_message("user"):
-            st.write(prompt)
-        
-        # Display assistant response with streaming
-        with st.chat_message("assistant"):
-            response_placeholder = st.empty()
-            full_response = ""
-            
-            # Stream the response
-            for response_chunk in stream_chat(prompt):
-                full_response += response_chunk
-                response_placeholder.markdown(full_response + "▌")
-            response_placeholder.markdown(full_response)
-        
-        # Add assistant response to chat history
-        st.session_state.messages.append({"role": "assistant", "content": full_response})
+            label="Top-p (nucleus sampling)",
+        ),
+    ],
+)
+
 
 if __name__ == "__main__":
-    main()
+    demo.launch()