streaming functions

src/pages/api/stream.js

@@ -20,6 +20,7 @@ export const LLMStream = async (
   messages,
   functions
 ) => {
+
   let url = `${OPENAI_API_HOST}/v1/chat/completions`;
   const res = await fetch(url, {
     headers: {
@@ -40,6 +41,7 @@ export const LLMStream = async (
       ],
       max_tokens: 1000,
       temperature: temperature,
+      functions: [functions['googleCustomSearch']['googleCustomSearchSchema']],
       stream: true,
     }),
   });
@@ -67,6 +69,11 @@ export const LLMStream = async (
 
   const stream = new ReadableStream({
     async start(controller) {
+      let func_call = {
+        "name": null,
+        "arguments": "",
+      };
+  
       const onParse = async (event) => {
         if (event.type === 'event') {
           const data = event.data;
@@ -75,38 +82,64 @@ export const LLMStream = async (
             if (data === "[DONE]" || !data) {
               return;
             }
+  
             const json = JSON.parse(data);
-            if (json.choices[0].finish_reason === "stop") {
-              controller.close();
-              return;
-            } else if (json.choices[0].finish_reason === "function_call") {
-              console.log('function called', json);
-              const fnName = json.choices[0].message.function_call.name;
-              const args = json.choices[0].message.function_call.arguments;
-          
-              const fn = functions[fnName];
-              const result = await fn(...Object.values(JSON.parse(args)));
-          
-              console.log(`Function call: ${fnName}, Arguments: ${args}`);
-              console.log(`Calling Function ${fnName} Result: ` + result);
+            console.log(json);  
+  
+            if (Array.isArray(json.choices) && json.choices.length > 0) {
+              const choice = json.choices[0];
+              console.log(choice);
+              const delta = choice.delta;
+  
+              if (choice.finish_reason === "stop") {
+                controller.close();
+                return;
+              }
+  
+              if (delta.hasOwnProperty("function_call")) {
+                if (delta.function_call.hasOwnProperty("name")) {
+                  func_call["name"] = delta.function_call["name"];
+                }
+                if (delta.function_call.hasOwnProperty("arguments")) {
+                  func_call["arguments"] += delta.function_call["arguments"];
+                }
+              }
+  
+              if (choice.finish_reason === "function_call") {
+                // function call here using func_call
+                console.log('Function call:', func_call);
+                const fn = functions[func_call.name]['googleCustomSearch'];
+                console.log(fn);
+                const funcResult = await fn(JSON.parse(func_call.arguments));
+                console.log(funcResult);
+                const serpQueue = encoder.encode(funcResult);
+                controller.enqueue(serpQueue);
+                return;
+              }
+  
+              if (delta && 'content' in delta) {
+                const text = delta.content;
+                const queue = encoder.encode(text);
+                controller.enqueue(queue);
+              }
+            } else {
+              console.error('No choices found in json');
             }
-            const text = json.choices[0].delta.content;
-            const queue = encoder.encode(text);
-            controller.enqueue(queue);
           } catch (e) {
             console.log(e);
             controller.error(e);
           }
         }
       };
-
+  
       const parser = createParser(onParse);
-
+  
       for await (const chunk of res.body) {
         parser.feed(decoder.decode(chunk));
       }
     },
   });
+  
 
   return stream;
 };