⚡ (openai) Use Vercel's AI SDK for streaming

apps/viewer/package.json

@@ -19,6 +19,7 @@
     "@typebot.io/js": "workspace:*",
     "@typebot.io/prisma": "workspace:*",
     "@typebot.io/react": "workspace:*",
+    "ai": "^2.1.3",
     "aws-sdk": "2.1384.0",
     "bot-engine": "workspace:*",
     "cors": "2.8.5",
@@ -30,6 +31,7 @@
     "nextjs-cors": "2.1.2",
     "nodemailer": "6.9.2",
     "openai": "3.2.1",
+    "openai-edge": "^1.1.0",
     "qs": "6.11.2",
     "react": "18.2.0",
     "react-dom": "18.2.0",

apps/viewer/src/features/blocks/integrations/openai/getChatCompletionStream.ts

@@ -6,10 +6,12 @@ import {
   OpenAICredentials,
 } from '@typebot.io/schemas/features/blocks/integrations/openai'
 import { SessionState } from '@typebot.io/schemas/features/chat'
-import type {
+import { OpenAIStream } from 'ai'
+import {
   ChatCompletionRequestMessage,
-  CreateChatCompletionRequest,
+  Configuration,
-} from 'openai'
+  OpenAIApi,
+} from 'openai-edge'
 
 export const getChatCompletionStream =
   (conn: Connection) =>
@@ -37,19 +39,18 @@ export const getChatCompletionStream =
       options.advancedSettings?.temperature
     )
 
-    const res = await fetch('https://api.openai.com/v1/chat/completions', {
+    const config = new Configuration({
-      headers: {
+      apiKey,
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${apiKey}`,
-      },
-      method: 'POST',
-      body: JSON.stringify({
-        messages,
-        model: options.model,
-        temperature,
-        stream: true,
-      } satisfies CreateChatCompletionRequest),
     })
 
-    return res.body
+    const openai = new OpenAIApi(config)
+
+    const response = await openai.createChatCompletion({
+      model: options.model,
+      temperature,
+      stream: true,
+      messages,
+    })
+
+    return OpenAIStream(response)
   }

apps/viewer/src/pages/api/integrations/openai/streamer.ts

@@ -1,6 +1,7 @@
 import { getChatCompletionStream } from '@/features/blocks/integrations/openai/getChatCompletionStream'
 import { connect } from '@planetscale/database'
 import { IntegrationBlockType, SessionState } from '@typebot.io/schemas'
+import { StreamingTextResponse } from 'ai'
 import { ChatCompletionRequestMessage } from 'openai'
 
 export const config = {
@@ -15,8 +16,7 @@ const handler = async (req: Request) => {
         'Access-Control-Allow-Origin': '*',
         'Access-Control-Allow-Methods': 'POST',
         'Access-Control-Expose-Headers': 'Content-Length, X-JSON',
-        'Access-Control-Allow-Headers':
+        'Access-Control-Allow-Headers': '*',
-          'apikey,X-Client-Info, Content-Type, Authorization, Accept, Accept-Language, X-Authorization',
       },
     })
   }
@@ -66,12 +66,10 @@ const handler = async (req: Request) => {
     messages
   )
 
-  if (!stream) return new Response('Missing credentials', { status: 400 })
+  if (!stream) return new Response('Could not create stream', { status: 400 })
 
-  return new Response(stream, {
+  return new StreamingTextResponse(stream, {
-    status: 200,
     headers: {
-      'Content-Type': 'application/json; charset=utf-8',
       'Access-Control-Allow-Origin': '*',
     },
   })

packages/embeds/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/js",
-  "version": "0.0.64",
+  "version": "0.0.65",
   "description": "Javascript library to display typebots on your website",
   "type": "module",
   "main": "dist/index.js",

packages/embeds/js/src/features/blocks/integrations/openai/streamChat.ts

@@ -1,10 +1,8 @@
-import { getOpenAiStreamerQuery } from '@/queries/getOpenAiStreamerQuery'
 import { ClientSideActionContext } from '@/types'
-import {
+import { guessApiHost } from '@/utils/guessApiHost'
-  createParser,
+import { isNotEmpty } from '@typebot.io/lib/utils'
-  ParsedEvent,
+
-  ReconnectInterval,
+let abortController: AbortController | null = null
-} from 'eventsource-parser'
 
 export const streamChat =
   (context: ClientSideActionContext) =>
@@ -13,59 +11,76 @@ export const streamChat =
       content?: string | undefined
       role?: 'system' | 'user' | 'assistant' | undefined
     }[],
-    {
+    { onStreamedMessage }: { onStreamedMessage?: (message: string) => void }
-      onStreamedMessage,
-      isRetrying,
-    }: { onStreamedMessage?: (message: string) => void; isRetrying?: boolean }
   ): Promise<{ message?: string; error?: object }> => {
-    const data = await getOpenAiStreamerQuery(context)(messages)
+    try {
+      abortController = new AbortController()
 
-    if (!data) return { error: { message: "Couldn't get streamer data" } }
+      const apiHost = context.apiHost
 
-    let message = ''
+      const res = await fetch(
+        `${
+          isNotEmpty(apiHost) ? apiHost : guessApiHost()
+        }/api/integrations/openai/streamer`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({
+            sessionId: context.sessionId,
+            messages,
+          }),
+          signal: abortController.signal,
+        }
+      )
 
-    const reader = data.getReader()
+      if (!res.ok) {
-    const decoder = new TextDecoder()
+        return {
-
+          error: {
-    const onParse = (event: ParsedEvent | ReconnectInterval) => {
+            message: (await res.text()) || 'Failed to fetch the chat response.',
-      if (event.type === 'event') {
+          },
-        const data = event.data
-        try {
-          const json = JSON.parse(data) as {
-            choices: { delta: { content: string } }[]
-          }
-          const text = json.choices.at(0)?.delta.content
-          if (!text) return
-          message += text
-          onStreamedMessage?.(message)
-        } catch (e) {
-          console.error(e)
         }
       }
-    }
 
-    const parser = createParser(onParse)
+      if (!res.body) {
-
+        throw new Error('The response body is empty.')
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      const { value, done } = await reader.read()
-      if (done || !value) break
-      const dataString = decoder.decode(value)
-      if (dataString.includes('503 Service Temporarily Unavailable')) {
-        if (isRetrying)
-          return { error: { message: "Couldn't get streamer data" } }
-        await new Promise((resolve) => setTimeout(resolve, 3000))
-        return streamChat(context)(messages, {
-          onStreamedMessage,
-          isRetrying: true,
-        })
       }
-      if (dataString.includes('[DONE]')) break
-      if (dataString.includes('"error":')) {
-        return { error: JSON.parse(dataString).error }
-      }
-      parser.feed(dataString)
-    }
 
-    return { message }
+      let message = ''
+
+      const reader = res.body.getReader()
+      const decoder = new TextDecoder()
+
+      // eslint-disable-next-line no-constant-condition
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) {
+          break
+        }
+        const chunk = decoder.decode(value)
+        if (onStreamedMessage) onStreamedMessage(chunk)
+        message += chunk
+        if (abortController === null) {
+          reader.cancel()
+          break
+        }
+      }
+
+      abortController = null
+
+      return { message }
+    } catch (err) {
+      console.error(err)
+      // Ignore abort errors as they are expected.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      if ((err as any).name === 'AbortError') {
+        abortController = null
+        return { error: { message: 'Request aborted' } }
+      }
+
+      if (err instanceof Error) return { error: { message: err.message } }
+
+      return { error: { message: 'Failed to fetch the chat response.' } }
+    }
   }

packages/embeds/react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/react",
-  "version": "0.0.64",
+  "version": "0.0.65",
   "description": "React library to display typebots on your website",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

pnpm-lock.yaml

@@ -517,6 +517,9 @@ importers:
       '@typebot.io/react':
         specifier: workspace:*
         version: link:../../packages/embeds/react
+      ai:
+        specifier: ^2.1.3
+        version: 2.1.3(react@18.2.0)(svelte@3.59.1)(vue@3.3.4)
       aws-sdk:
         specifier: 2.1384.0
         version: 2.1384.0
@@ -550,6 +553,9 @@ importers:
       openai:
         specifier: 3.2.1
         version: 3.2.1
+      openai-edge:
+        specifier: ^1.1.0
+        version: 1.1.0
       qs:
         specifier: 6.11.2
         version: 6.11.2
@@ -9034,6 +9040,89 @@ packages:
       - supports-color
     dev: false
 
+  /@vue/compiler-core@3.3.4:
+    resolution: {integrity: sha512-cquyDNvZ6jTbf/+x+AgM2Arrp6G4Dzbb0R64jiG804HRMfRiFXWI6kqUVqZ6ZR0bQhIoQjB4+2bhNtVwndW15g==}
+    dependencies:
+      '@babel/parser': 7.22.3
+      '@vue/shared': 3.3.4
+      estree-walker: 2.0.2
+      source-map-js: 1.0.2
+    dev: false
+
+  /@vue/compiler-dom@3.3.4:
+    resolution: {integrity: sha512-wyM+OjOVpuUukIq6p5+nwHYtj9cFroz9cwkfmP9O1nzH68BenTTv0u7/ndggT8cIQlnBeOo6sUT/gvHcIkLA5w==}
+    dependencies:
+      '@vue/compiler-core': 3.3.4
+      '@vue/shared': 3.3.4
+    dev: false
+
+  /@vue/compiler-sfc@3.3.4:
+    resolution: {integrity: sha512-6y/d8uw+5TkCuzBkgLS0v3lSM3hJDntFEiUORM11pQ/hKvkhSKZrXW6i69UyXlJQisJxuUEJKAWEqWbWsLeNKQ==}
+    dependencies:
+      '@babel/parser': 7.22.3
+      '@vue/compiler-core': 3.3.4
+      '@vue/compiler-dom': 3.3.4
+      '@vue/compiler-ssr': 3.3.4
+      '@vue/reactivity-transform': 3.3.4
+      '@vue/shared': 3.3.4
+      estree-walker: 2.0.2
+      magic-string: 0.30.0
+      postcss: 8.4.23
+      source-map-js: 1.0.2
+    dev: false
+
+  /@vue/compiler-ssr@3.3.4:
+    resolution: {integrity: sha512-m0v6oKpup2nMSehwA6Uuu+j+wEwcy7QmwMkVNVfrV9P2qE5KshC6RwOCq8fjGS/Eak/uNb8AaWekfiXxbBB6gQ==}
+    dependencies:
+      '@vue/compiler-dom': 3.3.4
+      '@vue/shared': 3.3.4
+    dev: false
+
+  /@vue/reactivity-transform@3.3.4:
+    resolution: {integrity: sha512-MXgwjako4nu5WFLAjpBnCj/ieqcjE2aJBINUNQzkZQfzIZA4xn+0fV1tIYBJvvva3N3OvKGofRLvQIwEQPpaXw==}
+    dependencies:
+      '@babel/parser': 7.22.3
+      '@vue/compiler-core': 3.3.4
+      '@vue/shared': 3.3.4
+      estree-walker: 2.0.2
+      magic-string: 0.30.0
+    dev: false
+
+  /@vue/reactivity@3.3.4:
+    resolution: {integrity: sha512-kLTDLwd0B1jG08NBF3R5rqULtv/f8x3rOFByTDz4J53ttIQEDmALqKqXY0J+XQeN0aV2FBxY8nJDf88yvOPAqQ==}
+    dependencies:
+      '@vue/shared': 3.3.4
+    dev: false
+
+  /@vue/runtime-core@3.3.4:
+    resolution: {integrity: sha512-R+bqxMN6pWO7zGI4OMlmvePOdP2c93GsHFM/siJI7O2nxFRzj55pLwkpCedEY+bTMgp5miZ8CxfIZo3S+gFqvA==}
+    dependencies:
+      '@vue/reactivity': 3.3.4
+      '@vue/shared': 3.3.4
+    dev: false
+
+  /@vue/runtime-dom@3.3.4:
+    resolution: {integrity: sha512-Aj5bTJ3u5sFsUckRghsNjVTtxZQ1OyMWCr5dZRAPijF/0Vy4xEoRCwLyHXcj4D0UFbJ4lbx3gPTgg06K/GnPnQ==}
+    dependencies:
+      '@vue/runtime-core': 3.3.4
+      '@vue/shared': 3.3.4
+      csstype: 3.1.2
+    dev: false
+
+  /@vue/server-renderer@3.3.4(vue@3.3.4):
+    resolution: {integrity: sha512-Q6jDDzR23ViIb67v+vM1Dqntu+HUexQcsWKhhQa4ARVzxOY2HbC7QRW/ggkDBd5BU+uM1sV6XOAP0b216o34JQ==}
+    peerDependencies:
+      vue: 3.3.4
+    dependencies:
+      '@vue/compiler-ssr': 3.3.4
+      '@vue/shared': 3.3.4
+      vue: 3.3.4
+    dev: false
+
+  /@vue/shared@3.3.4:
+    resolution: {integrity: sha512-7OjdcV8vQ74eiz1TZLzZP4JwqM5fA94K6yntPS5Z25r9HDuGNzaGdgvwKYq6S+MxwF0TFRwe50fIR/MYnakdkQ==}
+    dev: false
+
   /@webassemblyjs/ast@1.11.1:
     resolution: {integrity: sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==}
     dependencies:
@@ -9241,6 +9330,31 @@ packages:
       indent-string: 4.0.0
     dev: false
 
+  /ai@2.1.3(react@18.2.0)(svelte@3.59.1)(vue@3.3.4):
+    resolution: {integrity: sha512-TwSSYf7YZdwTc9ZdV0qnvToesEePVvH0UlzYmLY44uOHkhNYYsPw5wh7Y+GwS/mHFBnPOm7NFDZbtFGGn+D0mQ==}
+    engines: {node: '>=14.6'}
+    peerDependencies:
+      react: ^18.0.0
+      svelte: ^3.29.0
+      vue: ^3.3.4
+    peerDependenciesMeta:
+      react:
+        optional: true
+      svelte:
+        optional: true
+      vue:
+        optional: true
+    dependencies:
+      eventsource-parser: 1.0.0
+      nanoid: 3.3.6
+      react: 18.2.0
+      sswr: 1.10.0(svelte@3.59.1)
+      svelte: 3.59.1
+      swr: 2.1.5(react@18.2.0)
+      swrv: 1.0.3(vue@3.3.4)
+      vue: 3.3.4
+    dev: false
+
   /ajv-formats@2.1.1(ajv@8.12.0):
     resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==}
     peerDependencies:
@@ -15508,6 +15622,13 @@ packages:
       '@jridgewell/sourcemap-codec': 1.4.15
     dev: false
 
+  /magic-string@0.30.0:
+    resolution: {integrity: sha512-LA+31JYDJLs82r2ScLrlz1GjSgu66ZV518eyWT+S8VhyQn/JL0u9MeBOvQMGYiPk1DBiSN9DDMOcXvigJZaViQ==}
+    engines: {node: '>=12'}
+    dependencies:
+      '@jridgewell/sourcemap-codec': 1.4.15
+    dev: false
+
   /make-dir@3.1.0:
     resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
     engines: {node: '>=8'}
@@ -16614,6 +16735,11 @@ packages:
       is-wsl: 2.2.0
     dev: false
 
+  /openai-edge@1.1.0:
+    resolution: {integrity: sha512-VkXT7yhKjr1QlC/wIMuPW/s003cPHUtiNjt/kH+54iKDZZ7eiGMPuymhpSWJ/zVZfVz6+94okGd6rNuu7JRBDw==}
+    engines: {node: '>=12'}
+    dev: false
+
   /openai@3.2.1:
     resolution: {integrity: sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==}
     dependencies:
@@ -19451,6 +19577,15 @@ packages:
       number-is-nan: 1.0.1
     dev: false
 
+  /sswr@1.10.0(svelte@3.59.1):
+    resolution: {integrity: sha512-nLWAJSQy3h8t7rrbTXanRyVHuQPj4PwKIVGe4IMlxJFdhyaxnN/JGACnvQKGDeWiTGYIZIx/jRuUsPEF0867Pg==}
+    peerDependencies:
+      svelte: ^3.29.0
+    dependencies:
+      svelte: 3.59.1
+      swrev: 3.0.0
+    dev: false
+
   /stable@0.1.8:
     resolution: {integrity: sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==}
     deprecated: 'Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility'
@@ -19824,6 +19959,11 @@ packages:
     resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
     engines: {node: '>= 0.4'}
 
+  /svelte@3.59.1:
+    resolution: {integrity: sha512-pKj8fEBmqf6mq3/NfrB9SLtcJcUvjYSWyePlfCqN9gujLB25RitWK8PvFzlwim6hD/We35KbPlRteuA6rnPGcQ==}
+    engines: {node: '>= 8'}
+    dev: false
+
   /svg-parser@2.0.4:
     resolution: {integrity: sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==}
     dev: false
@@ -19856,6 +19996,18 @@ packages:
       use-sync-external-store: 1.2.0(react@18.2.0)
     dev: false
 
+  /swrev@3.0.0:
+    resolution: {integrity: sha512-QJuZiptdOmbDY45pECBRVEgnoBlOKjeT2MWVz04wKHpWX15hM3P7EjcIbHDg5yLoPCMQ7to3349MEE+l9QF5HA==}
+    dev: false
+
+  /swrv@1.0.3(vue@3.3.4):
+    resolution: {integrity: sha512-sl+eLEE+aPPjhP1E8gQ75q3RPRyw5Gd/kROnrTFo3+LkCeLskv7F+uAl5W97wgJkzitobL6FLsRPVm0DgIgN8A==}
+    peerDependencies:
+      vue: '>=3.2.26 < 4'
+    dependencies:
+      vue: 3.3.4
+    dev: false
+
   /symbol-observable@1.0.1:
     resolution: {integrity: sha512-Kb3PrPYz4HanVF1LVGuAdW6LoVgIwjUYJGzFe7NDrBLCN4lsV/5J0MFurV+ygS4bRVwrCEt2c7MQ1R2a72oJDw==}
     engines: {node: '>=0.10.0'}
@@ -20992,6 +21144,16 @@ packages:
       fsevents: 2.3.2
     dev: false
 
+  /vue@3.3.4:
+    resolution: {integrity: sha512-VTyEYn3yvIeY1Py0WaYGZsXnz3y5UnGi62GjVEqvEGPl6nxbOrCXbVOTQWBEJUqAyTUk2uJ5JLVnYJ6ZzGbrSw==}
+    dependencies:
+      '@vue/compiler-dom': 3.3.4
+      '@vue/compiler-sfc': 3.3.4
+      '@vue/runtime-dom': 3.3.4
+      '@vue/server-renderer': 3.3.4(vue@3.3.4)
+      '@vue/shared': 3.3.4
+    dev: false
+
   /w3c-keyname@2.2.7:
     resolution: {integrity: sha512-XB8aa62d4rrVfoZYQaYNy3fy+z4nrfy2ooea3/0BnBzXW0tSdZ+lRgjzBZhk0La0H6h8fVyYCxx/qkQcAIuvfg==}
     dev: false