⚡ (openai) Stream chat completion to avoid serverless timeout (#526)

Closes #520

apps/builder/public/templates/basic-chat-gpt.json

@@ -55,11 +55,6 @@
             "task": "Create chat completion",
             "model": "gpt-3.5-turbo",
             "messages": [
-              {
-                "id": "fxg16pnlnwuhfpz1r51xslbd",
-                "role": "system",
-                "content": "You are ChatGPT, a large language model trained by OpenAI."
-              },
               {
                 "id": "vexqydoltfc5fkdrcednlvjz",
                 "role": "Messages sequence ✨",

apps/builder/src/features/blocks/integrations/googleSheets/googleSheets.spec.ts

@@ -143,7 +143,7 @@ test.describe.parallel('Google sheets integration', () => {
       .press('Enter')
     await expect(
       page.locator('typebot-standard').locator('text=Your name is:')
-    ).toHaveText(`Your name is: Georges2 Smith2`)
+    ).toHaveText(`Your name is: Georges2 Last name`)
   })
 })
 

apps/builder/src/features/credentials/api/createCredentials.ts

@@ -51,7 +51,7 @@ export const createCredentials = authenticatedProcedure
     if (!workspace)
       throw new TRPCError({ code: 'NOT_FOUND', message: 'Workspace not found' })
 
-    const { encryptedData, iv } = encrypt(credentials.data)
+    const { encryptedData, iv } = await encrypt(credentials.data)
     const createdCredentials = await prisma.credentials.create({
       data: {
         ...credentials,

apps/builder/src/features/editor/editor.spec.ts

@@ -174,8 +174,8 @@ test('Rename and icon change should work', async ({ page }) => {
   ])
 
   await page.goto(`/typebots/${typebotId}/edit`)
-
   await page.click('[data-testid="editable-icon"]')
+  await page.getByRole('button', { name: 'Emoji' }).click()
   await expect(page.locator('text="My awesome typebot"')).toBeVisible()
   await page.fill('input[placeholder="Search..."]', 'love')
   await page.click('text="😍"')

apps/builder/src/features/settings/settings.spec.ts

@@ -19,10 +19,7 @@ test.describe.parallel('Settings page', () => {
       await page.click('text="Typebot.io branding"')
       await expect(page.locator('a:has-text("Made with Typebot")')).toBeHidden()
 
-      await page.click('text="Remember session"')
-      await expect(
-        page.locator('input[type="checkbox"] >> nth=-3')
-      ).toHaveAttribute('checked', '')
+      await page.click('text="Remember user"')
 
       await expect(page.getByPlaceholder('Type your answer...')).toHaveValue(
         'Baptiste'

apps/builder/src/features/workspace/workspaces.spec.ts

@@ -74,6 +74,7 @@ test('can update workspace info', async ({ page }) => {
   await page.click('text=Settings & Members')
   await page.click('text="Settings"')
   await page.click('[data-testid="editable-icon"]')
+  await page.getByRole('button', { name: 'Emoji' }).click()
   await page.fill('input[placeholder="Search..."]', 'building')
   await page.click('text="🏦"')
   await page.waitForTimeout(500)
@@ -92,13 +93,13 @@ test('can manage members', async ({ page }) => {
     page.getByRole('heading', { name: 'Members (1/5)' })
   ).toBeVisible()
   await expect(page.locator('text="user@email.com"').nth(1)).toBeVisible()
-  await expect(page.locator('button >> text="Invite"')).toBeEnabled()
+  await expect(page.locator('button >> text="Invite"')).toBeDisabled()
   await page.fill(
     'input[placeholder="colleague@company.com"]',
     'guest@email.com'
   )
   await page.click('button >> text="Invite"')
-  await expect(page.locator('button >> text="Invite"')).toBeEnabled()
+  await expect(page.locator('button >> text="Invite"')).toBeVisible()
   await expect(
     page.locator('input[placeholder="colleague@company.com"]')
   ).toHaveAttribute('value', '')

apps/builder/src/lib/googleSheets.ts

@@ -21,10 +21,10 @@ export const getAuthenticatedGoogleClient = async (
     where: { id: credentialsId, workspace: { members: { some: { userId } } } },
   })) as CredentialsFromDb | undefined
   if (!credentials) return
-  const data = decrypt(
+  const data = (await decrypt(
     credentials.data,
     credentials.iv
-  ) as GoogleSheetsCredentials['data']
+  )) as GoogleSheetsCredentials['data']
 
   oauth2Client.setCredentials(data)
   oauth2Client.on('tokens', updateTokens(credentials.id, data))
@@ -47,7 +47,7 @@ const updateTokens =
       expiry_date: credentials.expiry_date,
       access_token: credentials.access_token,
     }
-    const { encryptedData, iv } = encrypt(newCredentials)
+    const { encryptedData, iv } = await encrypt(newCredentials)
     await prisma.credentials.update({
       where: { id: credentialsId },
       data: { data: encryptedData, iv },

apps/builder/src/pages/api/credentials.ts

@@ -28,7 +28,7 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => {
     const data = (
       typeof req.body === 'string' ? JSON.parse(req.body) : req.body
     ) as Credentials
-    const { encryptedData, iv } = encrypt(data.data)
+    const { encryptedData, iv } = await encrypt(data.data)
     const workspace = await prisma.workspace.findFirst({
       where: { id: workspaceId, members: { some: { userId: user.id } } },
       select: { id: true },

apps/builder/src/pages/api/credentials/google-sheets/callback.ts

@@ -37,7 +37,7 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => {
       return res
         .status(400)
         .send({ message: "User didn't accepted required scopes" })
-    const { encryptedData, iv } = encrypt(tokens)
+    const { encryptedData, iv } = await encrypt(tokens)
     const credentials = {
       name: email,
       type: 'google sheets',

apps/viewer/package.json

@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@dqbd/tiktoken": "^1.0.7",
+    "@planetscale/database": "^1.7.0",
     "@sentry/nextjs": "7.50.0",
     "@trpc/server": "10.23.0",
     "@typebot.io/js": "workspace:*",
@@ -22,6 +23,7 @@
     "aws-sdk": "2.1369.0",
     "bot-engine": "workspace:*",
     "cors": "2.8.5",
+    "eventsource-parser": "^1.0.0",
     "google-spreadsheet": "3.3.0",
     "got": "12.6.0",
     "libphonenumber-js": "1.10.28",

apps/viewer/src/features/blocks/inputs/payment/computePaymentInputRuntimeOptions.ts

@@ -88,7 +88,10 @@ const getStripeInfo = async (
     where: { id: credentialsId },
   })
   if (!credentials) return
-  return decrypt(credentials.data, credentials.iv) as StripeCredentials['data']
+  return (await decrypt(
+    credentials.data,
+    credentials.iv
+  )) as StripeCredentials['data']
 }
 
 // https://stripe.com/docs/currencies#zero-decimal

apps/viewer/src/features/blocks/integrations/openai/createChatCompletionOpenAI.ts

@@ -5,7 +5,6 @@ import {
   ChatReply,
   SessionState,
   Variable,
-  VariableWithUnknowValue,
   VariableWithValue,
 } from '@typebot.io/schemas'
 import {
@@ -13,17 +12,25 @@ import {
   OpenAICredentials,
   modelLimit,
 } from '@typebot.io/schemas/features/blocks/integrations/openai'
-import { OpenAIApi, Configuration, ChatCompletionRequestMessage } from 'openai'
-import { isDefined, byId, isNotEmpty, isEmpty } from '@typebot.io/lib'
-import { decrypt } from '@typebot.io/lib/api/encryption'
+import type {
+  ChatCompletionRequestMessage,
+  CreateChatCompletionRequest,
+  CreateChatCompletionResponse,
+} from 'openai'
+import { byId, isNotEmpty, isEmpty } from '@typebot.io/lib'
+import { decrypt, isCredentialsV2 } from '@typebot.io/lib/api/encryption'
 import { saveErrorLog } from '@/features/logs/saveErrorLog'
 import { updateVariables } from '@/features/variables/updateVariables'
 import { parseVariables } from '@/features/variables/parseVariables'
-import { saveSuccessLog } from '@/features/logs/saveSuccessLog'
 import { parseVariableNumber } from '@/features/variables/parseVariableNumber'
 import { encoding_for_model } from '@dqbd/tiktoken'
+import got from 'got'
+import { resumeChatCompletion } from './resumeChatCompletion'
+import { isPlaneteScale } from '@/helpers/api/isPlanetScale'
+import { isVercel } from '@/helpers/api/isVercel'
 
 const minTokenCompletion = 200
+const createChatEndpoint = 'https://api.openai.com/v1/chat/completions'
 
 export const createChatCompletionOpenAI = async (
   state: SessionState,
@@ -52,13 +59,10 @@ export const createChatCompletionOpenAI = async (
     console.error('Could not find credentials in database')
     return { outgoingEdgeId, logs: [noCredentialsError] }
   }
-  const { apiKey } = decrypt(
+  const { apiKey } = (await decrypt(
     credentials.data,
     credentials.iv
-  ) as OpenAICredentials['data']
-  const configuration = new Configuration({
-    apiKey,
-  })
+  )) as OpenAICredentials['data']
   const { variablesTransformedToList, messages } = parseMessages(
     newSessionState.typebot.variables,
     options.model
@@ -71,52 +75,39 @@ export const createChatCompletionOpenAI = async (
   )
 
   try {
-    const openai = new OpenAIApi(configuration)
-    const response = await openai.createChatCompletion({
-      model: options.model,
-      messages,
-      temperature,
-    })
-    const messageContent = response.data.choices.at(0)?.message?.content
-    const totalTokens = response.data.usage?.total_tokens
+    if (
+      isPlaneteScale() &&
+      isVercel() &&
+      isCredentialsV2(credentials) &&
+      newSessionState.isStreamEnabled
+    )
+      return {
+        clientSideActions: [{ streamOpenAiChatCompletion: { messages } }],
+        outgoingEdgeId,
+        newSessionState,
+      }
+    const response = await got
+      .post(createChatEndpoint, {
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+        },
+        json: {
+          model: options.model,
+          messages,
+          temperature,
+        } satisfies CreateChatCompletionRequest,
+      })
+      .json<CreateChatCompletionResponse>()
+    const messageContent = response.choices.at(0)?.message?.content
+    const totalTokens = response.usage?.total_tokens
     if (isEmpty(messageContent)) {
       console.error('OpenAI block returned empty message', response)
       return { outgoingEdgeId, newSessionState }
     }
-    const newVariables = options.responseMapping.reduce<
-      VariableWithUnknowValue[]
-    >((newVariables, mapping) => {
-      const existingVariable = newSessionState.typebot.variables.find(
-        byId(mapping.variableId)
-      )
-      if (!existingVariable) return newVariables
-      if (mapping.valueToExtract === 'Message content') {
-        newVariables.push({
-          ...existingVariable,
-          value: Array.isArray(existingVariable.value)
-            ? existingVariable.value.concat(messageContent)
-            : messageContent,
-        })
-      }
-      if (mapping.valueToExtract === 'Total tokens' && isDefined(totalTokens)) {
-        newVariables.push({
-          ...existingVariable,
-          value: totalTokens,
-        })
-      }
-      return newVariables
-    }, [])
-    if (newVariables.length > 0)
-      newSessionState = await updateVariables(newSessionState)(newVariables)
-    state.result &&
-      (await saveSuccessLog({
-        resultId: state.result.id,
-        message: 'OpenAI block successfully executed',
-      }))
-    return {
+    return resumeChatCompletion(newSessionState, {
+      options,
       outgoingEdgeId,
-      newSessionState,
-    }
+    })(messageContent, totalTokens)
   } catch (err) {
     const log: NonNullable<ChatReply['logs']>[number] = {
       status: 'error',

apps/viewer/src/features/blocks/integrations/openai/getChatCompletionStream.ts

@@ -0,0 +1,103 @@
+import { parseVariableNumber } from '@/features/variables/parseVariableNumber'
+import { Connection } from '@planetscale/database'
+import { decrypt } from '@typebot.io/lib/api/encryption'
+import {
+  ChatCompletionOpenAIOptions,
+  OpenAICredentials,
+} from '@typebot.io/schemas/features/blocks/integrations/openai'
+import { SessionState } from '@typebot.io/schemas/features/chat'
+import {
+  ParsedEvent,
+  ReconnectInterval,
+  createParser,
+} from 'eventsource-parser'
+import type {
+  ChatCompletionRequestMessage,
+  CreateChatCompletionRequest,
+} from 'openai'
+
+export const getChatCompletionStream =
+  (conn: Connection) =>
+  async (
+    state: SessionState,
+    options: ChatCompletionOpenAIOptions,
+    messages: ChatCompletionRequestMessage[]
+  ) => {
+    if (!options.credentialsId) return
+    const credentials = (
+      await conn.execute('select data, iv from Credentials where id=?', [
+        options.credentialsId,
+      ])
+    ).rows.at(0) as { data: string; iv: string } | undefined
+    if (!credentials) {
+      console.error('Could not find credentials in database')
+      return
+    }
+    const { apiKey } = (await decrypt(
+      credentials.data,
+      credentials.iv
+    )) as OpenAICredentials['data']
+
+    const temperature = parseVariableNumber(state.typebot.variables)(
+      options.advancedSettings?.temperature
+    )
+
+    const encoder = new TextEncoder()
+    const decoder = new TextDecoder()
+
+    let counter = 0
+
+    const res = await fetch('https://api.openai.com/v1/chat/completions', {
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${apiKey}`,
+      },
+      method: 'POST',
+      body: JSON.stringify({
+        messages,
+        model: options.model,
+        temperature,
+        stream: true,
+      } satisfies CreateChatCompletionRequest),
+    })
+
+    const stream = new ReadableStream({
+      async start(controller) {
+        function onParse(event: ParsedEvent | ReconnectInterval) {
+          if (event.type === 'event') {
+            const data = event.data
+            if (data === '[DONE]') {
+              controller.close()
+              return
+            }
+            try {
+              const json = JSON.parse(data) as {
+                choices: { delta: { content: string } }[]
+              }
+              const text = json.choices.at(0)?.delta.content
+              if (counter < 2 && (text?.match(/\n/) || []).length) {
+                return
+              }
+              const queue = encoder.encode(text)
+              controller.enqueue(queue)
+              counter++
+            } catch (e) {
+              controller.error(e)
+            }
+          }
+        }
+
+        // stream response (SSE) from OpenAI may be fragmented into multiple chunks
+        // this ensures we properly read chunks & invoke an event for each SSE event stream
+        const parser = createParser(onParse)
+
+        // https://web.dev/streams/#asynchronous-iteration
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        for await (const chunk of res.body as any) {
+          parser.feed(decoder.decode(chunk))
+        }
+      },
+    })
+
+    return stream
+  }

apps/viewer/src/features/blocks/integrations/openai/resumeChatCompletion.ts

@@ -0,0 +1,52 @@
+import { saveSuccessLog } from '@/features/logs/saveSuccessLog'
+import { updateVariables } from '@/features/variables/updateVariables'
+import { byId, isDefined } from '@typebot.io/lib'
+import { SessionState } from '@typebot.io/schemas'
+import { ChatCompletionOpenAIOptions } from '@typebot.io/schemas/features/blocks/integrations/openai'
+import { VariableWithUnknowValue } from '@typebot.io/schemas/features/typebot/variable'
+
+export const resumeChatCompletion =
+  (
+    state: SessionState,
+    {
+      outgoingEdgeId,
+      options,
+    }: { outgoingEdgeId?: string; options: ChatCompletionOpenAIOptions }
+  ) =>
+  async (message: string, totalTokens?: number) => {
+    let newSessionState = state
+    const newVariables = options.responseMapping.reduce<
+      VariableWithUnknowValue[]
+    >((newVariables, mapping) => {
+      const existingVariable = newSessionState.typebot.variables.find(
+        byId(mapping.variableId)
+      )
+      if (!existingVariable) return newVariables
+      if (mapping.valueToExtract === 'Message content') {
+        newVariables.push({
+          ...existingVariable,
+          value: Array.isArray(existingVariable.value)
+            ? existingVariable.value.concat(message)
+            : message,
+        })
+      }
+      if (mapping.valueToExtract === 'Total tokens' && isDefined(totalTokens)) {
+        newVariables.push({
+          ...existingVariable,
+          value: totalTokens,
+        })
+      }
+      return newVariables
+    }, [])
+    if (newVariables.length > 0)
+      newSessionState = await updateVariables(newSessionState)(newVariables)
+    state.result &&
+      (await saveSuccessLog({
+        resultId: state.result.id,
+        message: 'OpenAI block successfully executed',
+      }))
+    return {
+      outgoingEdgeId,
+      newSessionState,
+    }
+  }

apps/viewer/src/features/blocks/integrations/sendEmail/executeSendEmailBlock.tsx

@@ -193,7 +193,10 @@ const getEmailInfo = async (
     where: { id: credentialsId },
   })
   if (!credentials) return
-  return decrypt(credentials.data, credentials.iv) as SmtpCredentials['data']
+  return (await decrypt(
+    credentials.data,
+    credentials.iv
+  )) as SmtpCredentials['data']
 }
 
 const getEmailBody = async ({

apps/viewer/src/features/blocks/logic/setVariable/executeSetVariable.ts

@@ -61,6 +61,11 @@ const evaluateSetVariableExpression =
     const evaluating = parseVariables(variables, { fieldToParse: 'id' })(
       str.includes('return ') ? str : `return ${str}`
     )
+    console.log(
+      variables.map((v) => v.id),
+      ...variables.map((v) => parseGuessedValueType(v.value)),
+      evaluating
+    )
     try {
       const func = Function(...variables.map((v) => v.id), evaluating)
       return func(...variables.map((v) => parseGuessedValueType(v.value)))

apps/viewer/src/features/chat/api/sendMessage.ts

@@ -154,6 +154,7 @@ const startSession = async (startParams?: StartParams, userId?: string) => {
     },
     currentTypebotId: typebot.id,
     dynamicTheme: parseDynamicThemeInState(typebot.theme),
+    isStreamEnabled: startParams.isStreamEnabled,
   }
 
   const { messages, input, clientSideActions, newSessionState, logs } =

apps/viewer/src/features/chat/helpers/continueBotFlow.ts

@@ -9,12 +9,13 @@ import {
   ChatReply,
   InputBlock,
   InputBlockType,
+  IntegrationBlockType,
   LogicBlockType,
   ResultInSession,
   SessionState,
   SetVariableBlock,
 } from '@typebot.io/schemas'
-import { isInputBlock, isNotDefined, byId } from '@typebot.io/lib'
+import { isInputBlock, isNotDefined, byId, isDefined } from '@typebot.io/lib'
 import { executeGroup } from './executeGroup'
 import { getNextGroup } from './getNextGroup'
 import { validateEmail } from '@/features/blocks/inputs/email/validateEmail'
@@ -23,6 +24,8 @@ import { validatePhoneNumber } from '@/features/blocks/inputs/phone/validatePhon
 import { validateUrl } from '@/features/blocks/inputs/url/validateUrl'
 import { updateVariables } from '@/features/variables/updateVariables'
 import { parseVariables } from '@/features/variables/parseVariables'
+import { OpenAIBlock } from '@typebot.io/schemas/features/blocks/integrations/openai'
+import { resumeChatCompletion } from '@/features/blocks/integrations/openai/resumeChatCompletion'
 
 export const continueBotFlow =
   (state: SessionState) =>
@@ -57,6 +60,16 @@ export const continueBotFlow =
         }
         newSessionState = await updateVariables(state)([newVariable])
       }
+    } else if (
+      isDefined(reply) &&
+      block.type === IntegrationBlockType.OPEN_AI &&
+      block.options.task === 'Create chat completion'
+    ) {
+      const result = await resumeChatCompletion(state, {
+        options: block.options,
+        outgoingEdgeId: block.outgoingEdgeId,
+      })(reply)
+      newSessionState = result.newSessionState
     } else if (!isInputBlock(block))
       throw new TRPCError({
         code: 'INTERNAL_SERVER_ERROR',
@@ -236,7 +249,10 @@ const computeStorageUsed = async (reply: string) => {
 
 const getOutgoingEdgeId =
   ({ typebot: { variables } }: Pick<SessionState, 'typebot'>) =>
-  (block: InputBlock | SetVariableBlock, reply: string | null) => {
+  (
+    block: InputBlock | SetVariableBlock | OpenAIBlock,
+    reply: string | null
+  ) => {
     if (
       block.type === InputBlockType.CHOICE &&
       !block.options.isMultipleChoice &&

apps/viewer/src/features/chat/helpers/executeGroup.ts

@@ -73,6 +73,10 @@ export const executeGroup =
         : null
 
       if (!executionResponse) continue
+      if (executionResponse.logs)
+        logs = [...(logs ?? []), ...executionResponse.logs]
+      if (executionResponse.newSessionState)
+        newSessionState = executionResponse.newSessionState
       if (
         'clientSideActions' in executionResponse &&
         executionResponse.clientSideActions
@@ -83,7 +87,8 @@ export const executeGroup =
         ]
         if (
           executionResponse.clientSideActions?.find(
-            (action) => 'setVariable' in action
+            (action) =>
+              'setVariable' in action || 'streamOpenAiChatCompletion' in action
           )
         ) {
           return {
@@ -101,10 +106,6 @@ export const executeGroup =
         }
       }
 
-      if (executionResponse.logs)
-        logs = [...(logs ?? []), ...executionResponse.logs]
-      if (executionResponse.newSessionState)
-        newSessionState = executionResponse.newSessionState
       if (executionResponse.outgoingEdgeId) {
         nextEdgeId = executionResponse.outgoingEdgeId
         break

apps/viewer/src/helpers/api/isPlanetScale.ts

@@ -0,0 +1,2 @@
+export const isPlaneteScale = () =>
+  process.env.DATABASE_URL?.includes('pscale_pw')

apps/viewer/src/helpers/api/isVercel.ts

@@ -0,0 +1,3 @@
+import { isDefined } from '@typebot.io/lib/utils'
+
+export const isVercel = () => isDefined(process.env.NEXT_PUBLIC_VERCEL_ENV)

apps/viewer/src/lib/google-sheets.ts

@@ -12,10 +12,10 @@ export const getAuthenticatedGoogleClient = async (
     where: { id: credentialsId },
   })) as CredentialsFromDb | undefined
   if (!credentials) return
-  const data = decrypt(
+  const data = (await decrypt(
     credentials.data,
     credentials.iv
-  ) as GoogleSheetsCredentials['data']
+  )) as GoogleSheetsCredentials['data']
 
   const oauth2Client = new OAuth2Client(
     process.env.GOOGLE_CLIENT_ID,
@@ -43,7 +43,7 @@ const updateTokens =
       expiry_date: credentials.expiry_date,
       access_token: credentials.access_token,
     }
-    const { encryptedData, iv } = encrypt(newCredentials)
+    const { encryptedData, iv } = await encrypt(newCredentials)
     await prisma.credentials.update({
       where: { id: credentialsId },
       data: { data: encryptedData, iv },

apps/viewer/src/pages/api/integrations/openai/streamer.ts

@@ -0,0 +1,78 @@
+import { getChatCompletionStream } from '@/features/blocks/integrations/openai/getChatCompletionStream'
+import { connect } from '@planetscale/database'
+import { IntegrationBlockType, SessionState } from '@typebot.io/schemas'
+import { ChatCompletionRequestMessage } from 'openai'
+
+export const config = {
+  runtime: 'edge',
+  regions: ['lhr1'],
+}
+
+const handler = async (req: Request) => {
+  if (req.method === 'OPTIONS') {
+    return new Response('ok', {
+      headers: {
+        'Access-Control-Allow-Origin': '*',
+        'Access-Control-Allow-Methods': 'POST',
+        'Access-Control-Expose-Headers': 'Content-Length, X-JSON',
+        'Access-Control-Allow-Headers':
+          'apikey,X-Client-Info, Content-Type, Authorization, Accept, Accept-Language, X-Authorization',
+      },
+    })
+  }
+  const { sessionId, messages } = (await req.json()) as {
+    sessionId: string
+    messages: ChatCompletionRequestMessage[]
+  }
+
+  if (!sessionId) return new Response('No session ID provided', { status: 400 })
+
+  if (!messages) return new Response('No messages provided', { status: 400 })
+
+  const conn = connect({ url: process.env.DATABASE_URL })
+
+  const chatSession = await conn.execute(
+    'select state from ChatSession where id=?',
+    [sessionId]
+  )
+
+  const state = (chatSession.rows.at(0) as { state: SessionState } | undefined)
+    ?.state
+
+  if (!state) return new Response('No state found', { status: 400 })
+
+  const group = state.typebot.groups.find(
+    (group) => group.id === state.currentBlock?.groupId
+  )
+  const blockIndex =
+    group?.blocks.findIndex(
+      (block) => block.id === state.currentBlock?.blockId
+    ) ?? -1
+
+  const block = blockIndex >= 0 ? group?.blocks[blockIndex ?? 0] : null
+
+  if (!block || !group)
+    return new Response('Current block not found', { status: 400 })
+
+  if (
+    block.type !== IntegrationBlockType.OPEN_AI ||
+    block.options.task !== 'Create chat completion'
+  )
+    return new Response('Current block is not an OpenAI block', { status: 400 })
+
+  const stream = await getChatCompletionStream(conn)(
+    state,
+    block.options,
+    messages
+  )
+
+  return new Response(stream, {
+    status: 200,
+    headers: {
+      'Content-Type': 'application/json; charset=utf-8',
+      'Access-Control-Allow-Origin': '*',
+    },
+  })
+}
+
+export default handler

apps/viewer/src/pages/api/integrations/stripe/createPaymentIntent.ts

@@ -112,7 +112,10 @@ const getStripeInfo = async (
     where: { id: credentialsId },
   })
   if (!credentials) return
-  return decrypt(credentials.data, credentials.iv) as StripeCredentials['data']
+  return (await decrypt(
+    credentials.data,
+    credentials.iv
+  )) as StripeCredentials['data']
 }
 
 // https://stripe.com/docs/currencies#zero-decimal

apps/viewer/src/pages/api/typebots/[typebotId]/integrations/email.tsx

@@ -170,7 +170,10 @@ const getEmailInfo = async (
     where: { id: credentialsId },
   })
   if (!credentials) return
-  return decrypt(credentials.data, credentials.iv) as SmtpCredentials['data']
+  return (await decrypt(
+    credentials.data,
+    credentials.iv
+  )) as SmtpCredentials['data']
 }
 
 const getEmailBody = async ({

apps/viewer/src/test/utils/databaseActions.ts

@@ -5,11 +5,11 @@ import { proWorkspaceId } from '@typebot.io/lib/playwright/databaseSetup'
 
 const prisma = new PrismaClient()
 
-export const createSmtpCredentials = (
+export const createSmtpCredentials = async (
   id: string,
   smtpData: SmtpCredentials['data']
 ) => {
-  const { encryptedData, iv } = encrypt(smtpData)
+  const { encryptedData, iv } = await encrypt(smtpData)
   return prisma.credentials.create({
     data: {
       id,

packages/embeds/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/js",
-  "version": "0.0.54",
+  "version": "0.0.55",
   "description": "Javascript library to display typebots on your website",
   "type": "module",
   "main": "dist/index.js",

packages/embeds/js/src/components/ConversationContainer/ChatChunk.tsx

@@ -11,7 +11,6 @@ type Props = Pick<ChatReply, 'messages' | 'input'> & {
   settings: Settings
   inputIndex: number
   context: BotContext
-  isLoadingBubbleDisplayed: boolean
   hasError: boolean
   hideAvatar: boolean
   onNewBubbleDisplayed: (blockId: string) => Promise<void>

packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx

@@ -69,7 +69,11 @@ export const ConversationContainer = (props: Props) => {
           (action) => isNotDefined(action.lastBubbleBlockId)
         )
         for (const action of actionsBeforeFirstBubble) {
-          const response = await executeClientSideAction(action)
+          if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+          const response = await executeClientSideAction(action, {
+            apiHost: props.context.apiHost,
+            sessionId: props.initialChatReply.sessionId,
+          })
           if (response && 'replyToSend' in response) {
             sendMessage(response.replyToSend)
             return
@@ -133,7 +137,11 @@ export const ConversationContainer = (props: Props) => {
         isNotDefined(action.lastBubbleBlockId)
       )
       for (const action of actionsBeforeFirstBubble) {
-        const response = await executeClientSideAction(action)
+        if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+        const response = await executeClientSideAction(action, {
+          apiHost: props.context.apiHost,
+          sessionId: props.initialChatReply.sessionId,
+        })
         if (response && 'replyToSend' in response) {
           sendMessage(response.replyToSend)
           return
@@ -174,7 +182,11 @@ export const ConversationContainer = (props: Props) => {
         (action) => action.lastBubbleBlockId === blockId
       )
       for (const action of actionsToExecute) {
-        const response = await executeClientSideAction(action)
+        if ('streamOpenAiChatCompletion' in action) setIsSending(true)
+        const response = await executeClientSideAction(action, {
+          apiHost: props.context.apiHost,
+          sessionId: props.initialChatReply.sessionId,
+        })
         if (response && 'replyToSend' in response) {
           sendMessage(response.replyToSend)
           return
@@ -200,7 +212,6 @@ export const ConversationContainer = (props: Props) => {
             input={chatChunk.input}
             theme={theme()}
             settings={props.initialChatReply.typebot.settings}
-            isLoadingBubbleDisplayed={isSending()}
             onNewBubbleDisplayed={handleNewBubbleDisplayed}
             onAllBubblesDisplayed={handleAllBubblesDisplayed}
             onSubmit={sendMessage}

packages/embeds/js/src/queries/getInitialChatReplyQuery.ts

@@ -26,6 +26,7 @@ export async function getInitialChatReplyQuery({
         prefilledVariables,
         startGroupId,
         resultId,
+        isStreamEnabled: true,
       },
     } satisfies SendMessageInput,
   })

packages/embeds/js/src/queries/getOpenAiStreamerQuery.ts

@@ -0,0 +1,33 @@
+import { guessApiHost } from '@/utils/guessApiHost'
+import { isNotEmpty } from '@typebot.io/lib'
+
+export const getOpenAiStreamerQuery =
+  ({ apiHost, sessionId }: { apiHost?: string; sessionId: string }) =>
+  async (
+    messages: {
+      content?: string | undefined
+      role?: 'system' | 'user' | 'assistant' | undefined
+    }[]
+  ) => {
+    const response = await fetch(
+      `${
+        isNotEmpty(apiHost) ? apiHost : guessApiHost()
+      }/api/integrations/openai/streamer`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          sessionId,
+          messages,
+        }),
+      }
+    )
+    if (!response.ok) {
+      throw new Error(response.statusText)
+    }
+
+    const data = response.body
+    return data
+  }

packages/embeds/js/src/utils/executeClientSideActions.ts

@@ -4,10 +4,18 @@ import { executeRedirect } from '@/features/blocks/logic/redirect'
 import { executeScript } from '@/features/blocks/logic/script/executeScript'
 import { executeSetVariable } from '@/features/blocks/logic/setVariable/executeSetVariable'
 import { executeWait } from '@/features/blocks/logic/wait/utils/executeWait'
+import { getOpenAiStreamerQuery } from '@/queries/getOpenAiStreamerQuery'
 import type { ChatReply } from '@typebot.io/schemas'
 
+type ClientSideActionContext = {
+  apiHost?: string
+  sessionId: string
+}
+
 export const executeClientSideAction = async (
-  clientSideAction: NonNullable<ChatReply['clientSideActions']>[0]
+  clientSideAction: NonNullable<ChatReply['clientSideActions']>[0],
+  context: ClientSideActionContext,
+  onStreamedMessage?: (message: string) => void
 ): Promise<
   { blockedPopupUrl: string } | { replyToSend: string | undefined } | void
 > => {
@@ -29,4 +37,41 @@ export const executeClientSideAction = async (
   if ('setVariable' in clientSideAction) {
     return executeSetVariable(clientSideAction.setVariable.scriptToExecute)
   }
+  if ('streamOpenAiChatCompletion' in clientSideAction) {
+    const text = await streamChat(context)(
+      clientSideAction.streamOpenAiChatCompletion.messages,
+      { onStreamedMessage }
+    )
+    return { replyToSend: text }
+  }
 }
+
+const streamChat =
+  (context: ClientSideActionContext) =>
+  async (
+    messages: {
+      content?: string | undefined
+      role?: 'system' | 'user' | 'assistant' | undefined
+    }[],
+    { onStreamedMessage }: { onStreamedMessage?: (message: string) => void }
+  ) => {
+    const data = await getOpenAiStreamerQuery(context)(messages)
+
+    if (!data) {
+      return
+    }
+
+    const reader = data.getReader()
+    const decoder = new TextDecoder()
+    let done = false
+
+    let message = ''
+    while (!done) {
+      const { value, done: doneReading } = await reader.read()
+      done = doneReading
+      const chunkValue = decoder.decode(value)
+      message += chunkValue
+      onStreamedMessage?.(message)
+    }
+    return message
+  }

packages/embeds/react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/react",
-  "version": "0.0.54",
+  "version": "0.0.55",
   "description": "React library to display typebots on your website",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/lib/api/encryption.ts

@@ -1,36 +1,75 @@
-import { randomBytes, createCipheriv, createDecipheriv } from 'crypto'
+import { Credentials } from '@typebot.io/schemas/features/credentials'
+import { decryptV1 } from './encryptionV1'
 
-const algorithm = 'aes-256-gcm'
+const algorithm = 'AES-GCM'
 const secretKey = process.env.ENCRYPTION_SECRET
 
-export const encrypt = (
+export const encrypt = async (
   data: object
-): { encryptedData: string; iv: string } => {
-  if (!secretKey) throw new Error(`ENCRYPTION_SECRET is not in environment`)
-  const iv = randomBytes(16)
-  const cipher = createCipheriv(algorithm, secretKey, iv)
-  const dataString = JSON.stringify(data)
-  const encryptedData =
-    cipher.update(dataString, 'utf8', 'hex') + cipher.final('hex')
-  const tag = cipher.getAuthTag()
+): Promise<{ encryptedData: string; iv: string }> => {
+  if (!secretKey) throw new Error('ENCRYPTION_SECRET is not in environment')
+  const iv = crypto.getRandomValues(new Uint8Array(12))
+  const encodedData = new TextEncoder().encode(JSON.stringify(data))
+
+  const key = await crypto.subtle.importKey(
+    'raw',
+    new TextEncoder().encode(secretKey),
+    algorithm,
+    false,
+    ['encrypt']
+  )
+
+  const encryptedBuffer = await crypto.subtle.encrypt(
+    { name: algorithm, iv },
+    key,
+    encodedData
+  )
+
+  const encryptedData = btoa(
+    String.fromCharCode.apply(null, Array.from(new Uint8Array(encryptedBuffer)))
+  )
+
+  const ivHex = Array.from(iv)
+    .map((byte) => byte.toString(16).padStart(2, '0'))
+    .join('')
+
   return {
     encryptedData,
-    iv: iv.toString('hex') + '.' + tag.toString('hex'),
+    iv: ivHex,
   }
 }
 
-export const decrypt = (encryptedData: string, auth: string): object => {
-  if (!secretKey) throw new Error(`ENCRYPTION_SECRET is not in environment`)
-  const [iv, tag] = auth.split('.')
-  const decipher = createDecipheriv(
+export const decrypt = async (
+  encryptedData: string,
+  ivHex: string
+): Promise<object> => {
+  if (ivHex.length !== 24) return decryptV1(encryptedData, ivHex)
+  if (!secretKey) throw new Error('ENCRYPTION_SECRET is not in environment')
+  const iv = new Uint8Array(
+    ivHex.match(/.{1,2}/g)?.map((byte) => parseInt(byte, 16)) ?? []
+  )
+
+  const key = await crypto.subtle.importKey(
+    'raw',
+    new TextEncoder().encode(secretKey),
     algorithm,
-    secretKey,
-    Buffer.from(iv, 'hex')
+    false,
+    ['decrypt']
   )
-  decipher.setAuthTag(Buffer.from(tag, 'hex'))
-  return JSON.parse(
-    (
-      decipher.update(Buffer.from(encryptedData, 'hex')) + decipher.final('hex')
-    ).toString()
+
+  const encryptedBuffer = new Uint8Array(
+    Array.from(atob(encryptedData)).map((char) => char.charCodeAt(0))
   )
+
+  const decryptedBuffer = await crypto.subtle.decrypt(
+    { name: algorithm, iv },
+    key,
+    encryptedBuffer
+  )
+
+  const decryptedData = new TextDecoder().decode(decryptedBuffer)
+  return JSON.parse(decryptedData)
 }
+
+export const isCredentialsV2 = (credentials: Pick<Credentials, 'iv'>) =>
+  credentials.iv.length === 24

packages/lib/api/encryptionV1.ts

@@ -0,0 +1,20 @@
+import { createDecipheriv } from 'crypto'
+
+const algorithm = 'aes-256-gcm'
+const secretKey = process.env.ENCRYPTION_SECRET
+
+export const decryptV1 = (encryptedData: string, auth: string): object => {
+  if (!secretKey) throw new Error(`ENCRYPTION_SECRET is not in environment`)
+  const [iv, tag] = auth.split('.')
+  const decipher = createDecipheriv(
+    algorithm,
+    secretKey,
+    Buffer.from(iv, 'hex')
+  )
+  decipher.setAuthTag(Buffer.from(tag, 'hex'))
+  return JSON.parse(
+    (
+      decipher.update(Buffer.from(encryptedData, 'hex')) + decipher.final('hex')
+    ).toString()
+  )
+}

packages/lib/playwright/databaseSetup.ts

@@ -125,8 +125,8 @@ export const setupUsers = async () => {
   })
 }
 
-const setupCredentials = () => {
-  const { encryptedData, iv } = encrypt({
+const setupCredentials = async () => {
+  const { encryptedData, iv } = await encrypt({
     expiry_date: 1642441058842,
     access_token:
       'ya29.A0ARrdaM--PV_87ebjywDJpXKb77NBFJl16meVUapYdfNv6W6ZzqqC47fNaPaRjbDbOIIcp6f49cMaX5ndK9TAFnKwlVqz3nrK9nLKqgyDIhYsIq47smcAIZkK56SWPx3X3DwAFqRu2UPojpd2upWwo-3uJrod',

packages/schemas/features/blocks/integrations/openai.ts

@@ -48,7 +48,7 @@ const initialOptionsSchema = z
   })
   .merge(openAIBaseOptionsSchema)
 
-const chatCompletionMessageSchema = z.object({
+export const chatCompletionMessageSchema = z.object({
   id: z.string(),
   role: z.enum(chatCompletionMessageRoles).optional(),
   content: z.string().optional(),

packages/schemas/features/chat.ts

@@ -1,4 +1,4 @@
-import { ZodDiscriminatedUnion, z } from 'zod'
+import { z } from 'zod'
 import {
   googleAnalyticsOptionsSchema,
   paymentInputRuntimeOptionsSchema,
@@ -17,6 +17,7 @@ import {
 import { answerSchema } from './answer'
 import { BubbleBlockType } from './blocks/bubbles/enums'
 import { inputBlockSchemas } from './blocks/schemas'
+import { chatCompletionMessageSchema } from './blocks/integrations/openai'
 
 const typebotInSessionStateSchema = publicTypebotSchema.pick({
   id: true,
@@ -62,6 +63,7 @@ export const sessionStateSchema = z.object({
       groupId: z.string(),
     })
     .optional(),
+  isStreamEnabled: z.boolean().optional(),
 })
 
 const chatSessionSchema = z.object({
@@ -162,6 +164,7 @@ const startParamsSchema = z.object({
     .describe(
       '[More info about prefilled variables.](https://docs.typebot.io/editor/variables#prefilled-variables)'
     ),
+  isStreamEnabled: z.boolean().optional(),
 })
 
 export const sendMessageInputSchema = z.object({
@@ -225,6 +228,15 @@ const clientSideActionSchema = z
           setVariable: z.object({ scriptToExecute: scriptToExecuteSchema }),
         })
       )
+      .or(
+        z.object({
+          streamOpenAiChatCompletion: z.object({
+            messages: z.array(
+              chatCompletionMessageSchema.pick({ content: true, role: true })
+            ),
+          }),
+        })
+      )
   )
 
 export const chatReplySchema = z.object({

pnpm-lock.yaml

@@ -502,6 +502,9 @@ importers:
       '@dqbd/tiktoken':
         specifier: ^1.0.7
         version: 1.0.7
+      '@planetscale/database':
+        specifier: ^1.7.0
+        version: 1.7.0
       '@sentry/nextjs':
         specifier: 7.50.0
         version: 7.50.0(next@13.3.4)(react@18.2.0)
@@ -526,6 +529,9 @@ importers:
       cors:
         specifier: 2.8.5
         version: 2.8.5
+      eventsource-parser:
+        specifier: ^1.0.0
+        version: 1.0.0
       google-spreadsheet:
         specifier: 3.3.0
         version: 3.3.0
@@ -7569,6 +7575,11 @@ packages:
       tslib: 2.5.0
     dev: false
 
+  /@planetscale/database@1.7.0:
+    resolution: {integrity: sha512-lWR6biXChUyQnxsT4RT1CIeR3ZJvwTQXiQ+158MnY3VjLwjHEGakDzdH9kwUGPk6CHvu6UeqRXp1DgUOVHJFTw==}
+    engines: {node: '>=16'}
+    dev: false
+
   /@playwright/test@1.33.0:
     resolution: {integrity: sha512-YunBa2mE7Hq4CfPkGzQRK916a4tuZoVx/EpLjeWlTVOnD4S2+fdaQZE0LJkbfhN5FTSKNLdcl7MoT5XB37bTkg==}
     engines: {node: '>=14'}
@@ -13627,6 +13638,11 @@ packages:
     resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
     engines: {node: '>=0.8.x'}
 
+  /eventsource-parser@1.0.0:
+    resolution: {integrity: sha512-9jgfSCa3dmEme2ES3mPByGXfgZ87VbP97tng1G2nWwWx6bV2nYxm2AWCrbQjXToSe+yYlqaZNtxffR9IeQr95g==}
+    engines: {node: '>=14.18'}
+    dev: false
+
   /evp_bytestokey@1.0.3:
     resolution: {integrity: sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==}
     dependencies: