Update llm.py

llm.py

@@ -31,7 +31,7 @@ import argparse
 model_path = "wizardLM-7B.ggml.q5_0.bin"
 
 
-def query_llm(index, prompt, service_context, retriever_mode='embedding', response_mode='tree_summarize'):
+def query_llm(index, prompt, service_context, retriever_mode='embedding', response_mode='compact'):
     response_synthesizer = ResponseSynthesizer.from_args(
         service_context=service_context,
         node_postprocessors=[
@@ -75,7 +75,7 @@ def construct_index(
     index_name,
     index_type,
     max_input_size=2048,
-    num_outputs=256,
+    num_outputs=2048,
     max_chunk_overlap=20,
     chunk_size_limit=None,
     embedding_limit=None,
@@ -87,16 +87,17 @@ def construct_index(
     embedding_limit = None if embedding_limit == 0 else embedding_limit
     separator = " " if separator == "" else separator
 
-    llm = LlamaCpp(model_path=model_path,
-        n_ctx=2048, 
+    llm = LlamaCpp(
+        model_path=model_path,
+        n_ctx=4096, 
         use_mlock=True,
         n_parts=-1, 
         temperature=0.7, 
         top_p=0.40,
-        last_n_tokens_size=200,
-        n_threads=4,
+        last_n_tokens_size=100,
+        n_threads=8,
         f16_kv=True,
-        max_tokens=400
+        max_tokens=150
     )
     llm_predictor = LLMPredictor(
         llm=llm
@@ -181,12 +182,12 @@ def ask_ai(
 ):
     logging.debug("Querying index...")
     prompt_helper = PromptHelper(
-        300,
-        200,
+        4096,
+        150,
         -20000
     )
     llm = LlamaCpp(model_path=model_path,
-        n_ctx=512, 
+        n_ctx=4096, 
         use_mlock=True,
         n_parts=-1, 
         temperature=temprature, 
@@ -216,11 +217,6 @@ def ask_ai(
     if response is not None:
         logging.info(f"Response: {response}")
         ret_text = response.response
-        ret_text += "\n----------\n"
-        nodes = []
-        for index, node in enumerate(response.source_nodes):
-            nodes.append(f"[{index+1}] {node.source_text}")
-        ret_text += "\n\n".join(nodes)
         return ret_text
     else:
         logging.debug("No response found, returning None")
@@ -230,15 +226,15 @@ def ask_ai(
 def search_construct(question, search_mode, index_select):
     print(f"You asked: {question}")
     llm = LlamaCpp(model_path=model_path,
-        n_ctx=500, 
+        n_ctx=400, 
         use_mlock=True,
         n_parts=-1, 
-        temperature=0.5, 
+        temperature=1, 
         top_p=0.40,
-        last_n_tokens_size=400,
-        n_threads=4,
+        last_n_tokens_size=100,
+        n_threads=6,
         f16_kv=True,
-        max_tokens=400
+        max_tokens=100
     )
     chat = llm
     search_terms = (