⚡ (openai) Stream chat completion to avoid serverless timeout (#526)

Closes #520

packages/embeds/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/js",
-  "version": "0.0.54",
+  "version": "0.0.55",
   "description": "Javascript library to display typebots on your website",
   "type": "module",
   "main": "dist/index.js",

packages/embeds/js/src/components/ConversationContainer/ChatChunk.tsx

@@ -11,7 +11,6 @@ type Props = Pick<ChatReply, 'messages' | 'input'> & {
   settings: Settings
   inputIndex: number
   context: BotContext
-  isLoadingBubbleDisplayed: boolean
   hasError: boolean
   hideAvatar: boolean
   onNewBubbleDisplayed: (blockId: string) => Promise<void>

packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx

@@ -69,7 +69,11 @@ export const ConversationContainer = (props: Props) => {
           (action) => isNotDefined(action.lastBubbleBlockId)
         )
         for (const action of actionsBeforeFirstBubble) {
-          const response = await executeClientSideAction(action)
+          if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+          const response = await executeClientSideAction(action, {
+            apiHost: props.context.apiHost,
+            sessionId: props.initialChatReply.sessionId,
+          })
           if (response && 'replyToSend' in response) {
             sendMessage(response.replyToSend)
             return
@@ -133,7 +137,11 @@ export const ConversationContainer = (props: Props) => {
         isNotDefined(action.lastBubbleBlockId)
       )
       for (const action of actionsBeforeFirstBubble) {
-        const response = await executeClientSideAction(action)
+        if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+        const response = await executeClientSideAction(action, {
+          apiHost: props.context.apiHost,
+          sessionId: props.initialChatReply.sessionId,
+        })
         if (response && 'replyToSend' in response) {
           sendMessage(response.replyToSend)
           return
@@ -174,7 +182,11 @@ export const ConversationContainer = (props: Props) => {
         (action) => action.lastBubbleBlockId === blockId
       )
       for (const action of actionsToExecute) {
-        const response = await executeClientSideAction(action)
+        if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+        const response = await executeClientSideAction(action, {
+          apiHost: props.context.apiHost,
+          sessionId: props.initialChatReply.sessionId,
+        })
         if (response && 'replyToSend' in response) {
           sendMessage(response.replyToSend)
           return
@@ -200,7 +212,6 @@ export const ConversationContainer = (props: Props) => {
             input={chatChunk.input}
             theme={theme()}
             settings={props.initialChatReply.typebot.settings}
-            isLoadingBubbleDisplayed={isSending()}
             onNewBubbleDisplayed={handleNewBubbleDisplayed}
             onAllBubblesDisplayed={handleAllBubblesDisplayed}
             onSubmit={sendMessage}

packages/embeds/js/src/queries/getInitialChatReplyQuery.ts

@@ -26,6 +26,7 @@ export async function getInitialChatReplyQuery({
         prefilledVariables,
         startGroupId,
         resultId,
+        isStreamEnabled: true,
       },
     } satisfies SendMessageInput,
   })

packages/embeds/js/src/queries/getOpenAiStreamerQuery.ts

@@ -0,0 +1,33 @@
+import { guessApiHost } from '@/utils/guessApiHost'
+import { isNotEmpty } from '@typebot.io/lib'
+
+export const getOpenAiStreamerQuery =
+  ({ apiHost, sessionId }: { apiHost?: string; sessionId: string }) =>
+  async (
+    messages: {
+      content?: string | undefined
+      role?: 'system' | 'user' | 'assistant' | undefined
+    }[]
+  ) => {
+    const response = await fetch(
+      `${
+        isNotEmpty(apiHost) ? apiHost : guessApiHost()
+      }/api/integrations/openai/streamer`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          sessionId,
+          messages,
+        }),
+      }
+    )
+    if (!response.ok) {
+      throw new Error(response.statusText)
+    }
+
+    const data = response.body
+    return data
+  }

packages/embeds/js/src/utils/executeClientSideActions.ts

@@ -4,10 +4,18 @@ import { executeRedirect } from '@/features/blocks/logic/redirect'
 import { executeScript } from '@/features/blocks/logic/script/executeScript'
 import { executeSetVariable } from '@/features/blocks/logic/setVariable/executeSetVariable'
 import { executeWait } from '@/features/blocks/logic/wait/utils/executeWait'
+import { getOpenAiStreamerQuery } from '@/queries/getOpenAiStreamerQuery'
 import type { ChatReply } from '@typebot.io/schemas'
 
+type ClientSideActionContext = {
+  apiHost?: string
+  sessionId: string
+}
+
 export const executeClientSideAction = async (
-  clientSideAction: NonNullable<ChatReply['clientSideActions']>[0]
+  clientSideAction: NonNullable<ChatReply['clientSideActions']>[0],
+  context: ClientSideActionContext,
+  onStreamedMessage?: (message: string) => void
 ): Promise<
   { blockedPopupUrl: string } | { replyToSend: string | undefined } | void
 > => {
@@ -29,4 +37,41 @@ export const executeClientSideAction = async (
   if ('setVariable' in clientSideAction) {
     return executeSetVariable(clientSideAction.setVariable.scriptToExecute)
   }
+  if ('streamOpenAiChatCompletion' in clientSideAction) {
+    const text = await streamChat(context)(
+      clientSideAction.streamOpenAiChatCompletion.messages,
+      { onStreamedMessage }
+    )
+    return { replyToSend: text }
+  }
 }
+
+const streamChat =
+  (context: ClientSideActionContext) =>
+  async (
+    messages: {
+      content?: string | undefined
+      role?: 'system' | 'user' | 'assistant' | undefined
+    }[],
+    { onStreamedMessage }: { onStreamedMessage?: (message: string) => void }
+  ) => {
+    const data = await getOpenAiStreamerQuery(context)(messages)
+
+    if (!data) {
+      return
+    }
+
+    const reader = data.getReader()
+    const decoder = new TextDecoder()
+    let done = false
+
+    let message = ''
+    while (!done) {
+      const { value, done: doneReading } = await reader.read()
+      done = doneReading
+      const chunkValue = decoder.decode(value)
+      message += chunkValue
+      onStreamedMessage?.(message)
+    }
+    return message
+  }

packages/embeds/react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/react",
-  "version": "0.0.54",
+  "version": "0.0.55",
   "description": "React library to display typebots on your website",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/lib/api/encryption.ts

@@ -1,36 +1,75 @@
-import { randomBytes, createCipheriv, createDecipheriv } from 'crypto'
+import { Credentials } from '@typebot.io/schemas/features/credentials'
+import { decryptV1 } from './encryptionV1'
 
-const algorithm = 'aes-256-gcm'
+const algorithm = 'AES-GCM'
 const secretKey = process.env.ENCRYPTION_SECRET
 
-export const encrypt = (
+export const encrypt = async (
   data: object
-): { encryptedData: string; iv: string } => {
-  if (!secretKey) throw new Error(`ENCRYPTION_SECRET is not in environment`)
-  const iv = randomBytes(16)
-  const cipher = createCipheriv(algorithm, secretKey, iv)
-  const dataString = JSON.stringify(data)
-  const encryptedData =
-    cipher.update(dataString, 'utf8', 'hex') + cipher.final('hex')
-  const tag = cipher.getAuthTag()
+): Promise<{ encryptedData: string; iv: string }> => {
+  if (!secretKey) throw new Error('ENCRYPTION_SECRET is not in environment')
+  const iv = crypto.getRandomValues(new Uint8Array(12))
+  const encodedData = new TextEncoder().encode(JSON.stringify(data))
+
+  const key = await crypto.subtle.importKey(
+    'raw',
+    new TextEncoder().encode(secretKey),
+    algorithm,
+    false,
+    ['encrypt']
+  )
+
+  const encryptedBuffer = await crypto.subtle.encrypt(
+    { name: algorithm, iv },
+    key,
+    encodedData
+  )
+
+  const encryptedData = btoa(
+    String.fromCharCode.apply(null, Array.from(new Uint8Array(encryptedBuffer)))
+  )
+
+  const ivHex = Array.from(iv)
+    .map((byte) => byte.toString(16).padStart(2, '0'))
+    .join('')
+
   return {
     encryptedData,
-    iv: iv.toString('hex') + '.' + tag.toString('hex'),
+    iv: ivHex,
   }
 }
 
-export const decrypt = (encryptedData: string, auth: string): object => {
-  if (!secretKey) throw new Error(`ENCRYPTION_SECRET is not in environment`)
-  const [iv, tag] = auth.split('.')
-  const decipher = createDecipheriv(
+export const decrypt = async (
+  encryptedData: string,
+  ivHex: string
+): Promise<object> => {
+  if (ivHex.length !== 24) return decryptV1(encryptedData, ivHex)
+  if (!secretKey) throw new Error('ENCRYPTION_SECRET is not in environment')
+  const iv = new Uint8Array(
+    ivHex.match(/.{1,2}/g)?.map((byte) => parseInt(byte, 16)) ?? []
+  )
+
+  const key = await crypto.subtle.importKey(
+    'raw',
+    new TextEncoder().encode(secretKey),
     algorithm,
-    secretKey,
-    Buffer.from(iv, 'hex')
+    false,
+    ['decrypt']
   )
-  decipher.setAuthTag(Buffer.from(tag, 'hex'))
-  return JSON.parse(
-    (
-      decipher.update(Buffer.from(encryptedData, 'hex')) + decipher.final('hex')
-    ).toString()
+
+  const encryptedBuffer = new Uint8Array(
+    Array.from(atob(encryptedData)).map((char) => char.charCodeAt(0))
   )
+
+  const decryptedBuffer = await crypto.subtle.decrypt(
+    { name: algorithm, iv },
+    key,
+    encryptedBuffer
+  )
+
+  const decryptedData = new TextDecoder().decode(decryptedBuffer)
+  return JSON.parse(decryptedData)
 }
+
+export const isCredentialsV2 = (credentials: Pick<Credentials, 'iv'>) =>
+  credentials.iv.length === 24

packages/lib/api/encryptionV1.ts

@@ -0,0 +1,20 @@
+import { createDecipheriv } from 'crypto'
+
+const algorithm = 'aes-256-gcm'
+const secretKey = process.env.ENCRYPTION_SECRET
+
+export const decryptV1 = (encryptedData: string, auth: string): object => {
+  if (!secretKey) throw new Error(`ENCRYPTION_SECRET is not in environment`)
+  const [iv, tag] = auth.split('.')
+  const decipher = createDecipheriv(
+    algorithm,
+    secretKey,
+    Buffer.from(iv, 'hex')
+  )
+  decipher.setAuthTag(Buffer.from(tag, 'hex'))
+  return JSON.parse(
+    (
+      decipher.update(Buffer.from(encryptedData, 'hex')) + decipher.final('hex')
+    ).toString()
+  )
+}

packages/lib/playwright/databaseSetup.ts

@@ -125,8 +125,8 @@ export const setupUsers = async () => {
   })
 }
 
-const setupCredentials = () => {
-  const { encryptedData, iv } = encrypt({
+const setupCredentials = async () => {
+  const { encryptedData, iv } = await encrypt({
     expiry_date: 1642441058842,
     access_token:
       'ya29.A0ARrdaM--PV_87ebjywDJpXKb77NBFJl16meVUapYdfNv6W6ZzqqC47fNaPaRjbDbOIIcp6f49cMaX5ndK9TAFnKwlVqz3nrK9nLKqgyDIhYsIq47smcAIZkK56SWPx3X3DwAFqRu2UPojpd2upWwo-3uJrod',

packages/schemas/features/blocks/integrations/openai.ts

@@ -48,7 +48,7 @@ const initialOptionsSchema = z
   })
   .merge(openAIBaseOptionsSchema)
 
-const chatCompletionMessageSchema = z.object({
+export const chatCompletionMessageSchema = z.object({
   id: z.string(),
   role: z.enum(chatCompletionMessageRoles).optional(),
   content: z.string().optional(),

packages/schemas/features/chat.ts

@@ -1,4 +1,4 @@
-import { ZodDiscriminatedUnion, z } from 'zod'
+import { z } from 'zod'
 import {
   googleAnalyticsOptionsSchema,
   paymentInputRuntimeOptionsSchema,
@@ -17,6 +17,7 @@ import {
 import { answerSchema } from './answer'
 import { BubbleBlockType } from './blocks/bubbles/enums'
 import { inputBlockSchemas } from './blocks/schemas'
+import { chatCompletionMessageSchema } from './blocks/integrations/openai'
 
 const typebotInSessionStateSchema = publicTypebotSchema.pick({
   id: true,
@@ -62,6 +63,7 @@ export const sessionStateSchema = z.object({
       groupId: z.string(),
     })
     .optional(),
+  isStreamEnabled: z.boolean().optional(),
 })
 
 const chatSessionSchema = z.object({
@@ -162,6 +164,7 @@ const startParamsSchema = z.object({
     .describe(
       '[More info about prefilled variables.](https://docs.typebot.io/editor/variables#prefilled-variables)'
     ),
+  isStreamEnabled: z.boolean().optional(),
 })
 
 export const sendMessageInputSchema = z.object({
@@ -225,6 +228,15 @@ const clientSideActionSchema = z
           setVariable: z.object({ scriptToExecute: scriptToExecuteSchema }),
         })
       )
+      .or(
+        z.object({
+          streamOpenAiChatCompletion: z.object({
+            messages: z.array(
+              chatCompletionMessageSchema.pick({ content: true, role: true })
+            ),
+          }),
+        })
+      )
   )
 
 export const chatReplySchema = z.object({