⚡ (openai) Stream chat completion to avoid serverless timeout (#526)

Closes #520

packages/embeds/js/src/components/ConversationContainer/ChatChunk.tsx

@@ -11,7 +11,6 @@ type Props = Pick<ChatReply, 'messages' | 'input'> & {
   settings: Settings
   inputIndex: number
   context: BotContext
-  isLoadingBubbleDisplayed: boolean
   hasError: boolean
   hideAvatar: boolean
   onNewBubbleDisplayed: (blockId: string) => Promise<void>

packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx

@@ -69,7 +69,11 @@ export const ConversationContainer = (props: Props) => {
           (action) => isNotDefined(action.lastBubbleBlockId)
         )
         for (const action of actionsBeforeFirstBubble) {
-          const response = await executeClientSideAction(action)
+          if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+          const response = await executeClientSideAction(action, {
+            apiHost: props.context.apiHost,
+            sessionId: props.initialChatReply.sessionId,
+          })
           if (response && 'replyToSend' in response) {
             sendMessage(response.replyToSend)
             return
@@ -133,7 +137,11 @@ export const ConversationContainer = (props: Props) => {
         isNotDefined(action.lastBubbleBlockId)
       )
       for (const action of actionsBeforeFirstBubble) {
-        const response = await executeClientSideAction(action)
+        if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+        const response = await executeClientSideAction(action, {
+          apiHost: props.context.apiHost,
+          sessionId: props.initialChatReply.sessionId,
+        })
         if (response && 'replyToSend' in response) {
           sendMessage(response.replyToSend)
           return
@@ -174,7 +182,11 @@ export const ConversationContainer = (props: Props) => {
         (action) => action.lastBubbleBlockId === blockId
       )
       for (const action of actionsToExecute) {
-        const response = await executeClientSideAction(action)
+        if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+        const response = await executeClientSideAction(action, {
+          apiHost: props.context.apiHost,
+          sessionId: props.initialChatReply.sessionId,
+        })
         if (response && 'replyToSend' in response) {
           sendMessage(response.replyToSend)
           return
@@ -200,7 +212,6 @@ export const ConversationContainer = (props: Props) => {
             input={chatChunk.input}
             theme={theme()}
             settings={props.initialChatReply.typebot.settings}
-            isLoadingBubbleDisplayed={isSending()}
             onNewBubbleDisplayed={handleNewBubbleDisplayed}
             onAllBubblesDisplayed={handleAllBubblesDisplayed}
             onSubmit={sendMessage}

packages/embeds/js/src/queries/getInitialChatReplyQuery.ts

@@ -26,6 +26,7 @@ export async function getInitialChatReplyQuery({
         prefilledVariables,
         startGroupId,
         resultId,
+        isStreamEnabled: true,
       },
     } satisfies SendMessageInput,
   })

packages/embeds/js/src/queries/getOpenAiStreamerQuery.ts

@@ -0,0 +1,33 @@
+import { guessApiHost } from '@/utils/guessApiHost'
+import { isNotEmpty } from '@typebot.io/lib'
+
+export const getOpenAiStreamerQuery =
+  ({ apiHost, sessionId }: { apiHost?: string; sessionId: string }) =>
+  async (
+    messages: {
+      content?: string | undefined
+      role?: 'system' | 'user' | 'assistant' | undefined
+    }[]
+  ) => {
+    const response = await fetch(
+      `${
+        isNotEmpty(apiHost) ? apiHost : guessApiHost()
+      }/api/integrations/openai/streamer`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          sessionId,
+          messages,
+        }),
+      }
+    )
+    if (!response.ok) {
+      throw new Error(response.statusText)
+    }
+
+    const data = response.body
+    return data
+  }

packages/embeds/js/src/utils/executeClientSideActions.ts

@@ -4,10 +4,18 @@ import { executeRedirect } from '@/features/blocks/logic/redirect'
 import { executeScript } from '@/features/blocks/logic/script/executeScript'
 import { executeSetVariable } from '@/features/blocks/logic/setVariable/executeSetVariable'
 import { executeWait } from '@/features/blocks/logic/wait/utils/executeWait'
+import { getOpenAiStreamerQuery } from '@/queries/getOpenAiStreamerQuery'
 import type { ChatReply } from '@typebot.io/schemas'
 
+type ClientSideActionContext = {
+  apiHost?: string
+  sessionId: string
+}
+
 export const executeClientSideAction = async (
-  clientSideAction: NonNullable<ChatReply['clientSideActions']>[0]
+  clientSideAction: NonNullable<ChatReply['clientSideActions']>[0],
+  context: ClientSideActionContext,
+  onStreamedMessage?: (message: string) => void
 ): Promise<
   { blockedPopupUrl: string } | { replyToSend: string | undefined } | void
 > => {
@@ -29,4 +37,41 @@ export const executeClientSideAction = async (
   if ('setVariable' in clientSideAction) {
     return executeSetVariable(clientSideAction.setVariable.scriptToExecute)
   }
+  if ('streamOpenAiChatCompletion' in clientSideAction) {
+    const text = await streamChat(context)(
+      clientSideAction.streamOpenAiChatCompletion.messages,
+      { onStreamedMessage }
+    )
+    return { replyToSend: text }
+  }
 }
+
+const streamChat =
+  (context: ClientSideActionContext) =>
+  async (
+    messages: {
+      content?: string | undefined
+      role?: 'system' | 'user' | 'assistant' | undefined
+    }[],
+    { onStreamedMessage }: { onStreamedMessage?: (message: string) => void }
+  ) => {
+    const data = await getOpenAiStreamerQuery(context)(messages)
+
+    if (!data) {
+      return
+    }
+
+    const reader = data.getReader()
+    const decoder = new TextDecoder()
+    let done = false
+
+    let message = ''
+    while (!done) {
+      const { value, done: doneReading } = await reader.read()
+      done = doneReading
+      const chunkValue = decoder.decode(value)
+      message += chunkValue
+      onStreamedMessage?.(message)
+    }
+    return message
+  }