From 63bebe9f83ff9d4f53c7601d600ba43dea73efab Mon Sep 17 00:00:00 2001 From: Baptiste Arnaud Date: Tue, 20 Jun 2023 17:23:14 +0200 Subject: [PATCH] :zap: (openai) Use Vercel's AI SDK for streaming --- apps/viewer/package.json | 2 + .../openai/getChatCompletionStream.ts | 33 ++-- .../pages/api/integrations/openai/streamer.ts | 23 ++- packages/embeds/js/package.json | 2 +- .../blocks/integrations/openai/streamChat.ts | 119 +++++++------ packages/embeds/react/package.json | 2 +- pnpm-lock.yaml | 162 ++++++++++++++++++ 7 files changed, 265 insertions(+), 78 deletions(-) diff --git a/apps/viewer/package.json b/apps/viewer/package.json index c4c3e6c9e..187a8dc77 100644 --- a/apps/viewer/package.json +++ b/apps/viewer/package.json @@ -19,6 +19,7 @@ "@typebot.io/js": "workspace:*", "@typebot.io/prisma": "workspace:*", "@typebot.io/react": "workspace:*", + "ai": "^2.1.3", "aws-sdk": "2.1384.0", "bot-engine": "workspace:*", "cors": "2.8.5", @@ -30,6 +31,7 @@ "nextjs-cors": "2.1.2", "nodemailer": "6.9.2", "openai": "3.2.1", + "openai-edge": "^1.1.0", "qs": "6.11.2", "react": "18.2.0", "react-dom": "18.2.0", diff --git a/apps/viewer/src/features/blocks/integrations/openai/getChatCompletionStream.ts b/apps/viewer/src/features/blocks/integrations/openai/getChatCompletionStream.ts index 87c5b93cd..19a96d838 100644 --- a/apps/viewer/src/features/blocks/integrations/openai/getChatCompletionStream.ts +++ b/apps/viewer/src/features/blocks/integrations/openai/getChatCompletionStream.ts @@ -6,10 +6,12 @@ import { OpenAICredentials, } from '@typebot.io/schemas/features/blocks/integrations/openai' import { SessionState } from '@typebot.io/schemas/features/chat' -import type { +import { OpenAIStream } from 'ai' +import { ChatCompletionRequestMessage, - CreateChatCompletionRequest, -} from 'openai' + Configuration, + OpenAIApi, +} from 'openai-edge' export const getChatCompletionStream = (conn: Connection) => @@ -37,19 +39,18 @@ export const getChatCompletionStream = options.advancedSettings?.temperature ) - const res = await fetch('https://api.openai.com/v1/chat/completions', { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, - }, - method: 'POST', - body: JSON.stringify({ - messages, - model: options.model, - temperature, - stream: true, - } satisfies CreateChatCompletionRequest), + const config = new Configuration({ + apiKey, }) - return res.body + const openai = new OpenAIApi(config) + + const response = await openai.createChatCompletion({ + model: options.model, + temperature, + stream: true, + messages, + }) + + return OpenAIStream(response) } diff --git a/apps/viewer/src/pages/api/integrations/openai/streamer.ts b/apps/viewer/src/pages/api/integrations/openai/streamer.ts index 99f8a913e..ffc458600 100644 --- a/apps/viewer/src/pages/api/integrations/openai/streamer.ts +++ b/apps/viewer/src/pages/api/integrations/openai/streamer.ts @@ -1,6 +1,7 @@ import { getChatCompletionStream } from '@/features/blocks/integrations/openai/getChatCompletionStream' import { connect } from '@planetscale/database' import { IntegrationBlockType, SessionState } from '@typebot.io/schemas' +import { StreamingTextResponse } from 'ai' import { ChatCompletionRequestMessage } from 'openai' export const config = { @@ -8,15 +9,23 @@ export const config = { regions: ['lhr1'], } +const allowedOrigins = [ + process.env.NEXT_PUBLIC_VIEWER_URL, + process.env.NEXTAUTH_URL, +] + const handler = async (req: Request) => { + const allowedOrigin = + allowedOrigins.find( + (origin) => origin && req.headers.get('Origin')?.startsWith(origin) + ) ?? 'null' if (req.method === 'OPTIONS') { return new Response('ok', { headers: { - 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Origin': allowedOrigin, 'Access-Control-Allow-Methods': 'POST', 'Access-Control-Expose-Headers': 'Content-Length, X-JSON', - 'Access-Control-Allow-Headers': - 'apikey,X-Client-Info, Content-Type, Authorization, Accept, Accept-Language, X-Authorization', + 'Access-Control-Allow-Headers': '*', }, }) } @@ -66,13 +75,11 @@ const handler = async (req: Request) => { messages ) - if (!stream) return new Response('Missing credentials', { status: 400 }) + if (!stream) return new Response('Could not create stream', { status: 400 }) - return new Response(stream, { - status: 200, + return new StreamingTextResponse(stream, { headers: { - 'Content-Type': 'application/json; charset=utf-8', - 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Origin': allowedOrigin, }, }) } diff --git a/packages/embeds/js/package.json b/packages/embeds/js/package.json index 348498db1..62c00d29d 100644 --- a/packages/embeds/js/package.json +++ b/packages/embeds/js/package.json @@ -1,6 +1,6 @@ { "name": "@typebot.io/js", - "version": "0.0.64", + "version": "0.0.65", "description": "Javascript library to display typebots on your website", "type": "module", "main": "dist/index.js", diff --git a/packages/embeds/js/src/features/blocks/integrations/openai/streamChat.ts b/packages/embeds/js/src/features/blocks/integrations/openai/streamChat.ts index 936329b74..2eef563b3 100644 --- a/packages/embeds/js/src/features/blocks/integrations/openai/streamChat.ts +++ b/packages/embeds/js/src/features/blocks/integrations/openai/streamChat.ts @@ -1,10 +1,8 @@ -import { getOpenAiStreamerQuery } from '@/queries/getOpenAiStreamerQuery' import { ClientSideActionContext } from '@/types' -import { - createParser, - ParsedEvent, - ReconnectInterval, -} from 'eventsource-parser' +import { guessApiHost } from '@/utils/guessApiHost' +import { isNotEmpty } from '@typebot.io/lib/utils' + +let abortController: AbortController | null = null export const streamChat = (context: ClientSideActionContext) => @@ -13,59 +11,76 @@ export const streamChat = content?: string | undefined role?: 'system' | 'user' | 'assistant' | undefined }[], - { - onStreamedMessage, - isRetrying, - }: { onStreamedMessage?: (message: string) => void; isRetrying?: boolean } + { onStreamedMessage }: { onStreamedMessage?: (message: string) => void } ): Promise<{ message?: string; error?: object }> => { - const data = await getOpenAiStreamerQuery(context)(messages) + try { + abortController = new AbortController() - if (!data) return { error: { message: "Couldn't get streamer data" } } + const apiHost = context.apiHost - let message = '' + const res = await fetch( + `${ + isNotEmpty(apiHost) ? apiHost : guessApiHost() + }/api/integrations/openai/streamer`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + sessionId: context.sessionId, + messages, + }), + signal: abortController.signal, + } + ) - const reader = data.getReader() - const decoder = new TextDecoder() - - const onParse = (event: ParsedEvent | ReconnectInterval) => { - if (event.type === 'event') { - const data = event.data - try { - const json = JSON.parse(data) as { - choices: { delta: { content: string } }[] - } - const text = json.choices.at(0)?.delta.content - if (!text) return - message += text - onStreamedMessage?.(message) - } catch (e) { - console.error(e) + if (!res.ok) { + return { + error: { + message: (await res.text()) || 'Failed to fetch the chat response.', + }, } } - } - const parser = createParser(onParse) - - // eslint-disable-next-line no-constant-condition - while (true) { - const { value, done } = await reader.read() - if (done || !value) break - const dataString = decoder.decode(value) - if (dataString.includes('503 Service Temporarily Unavailable')) { - if (isRetrying) - return { error: { message: "Couldn't get streamer data" } } - await new Promise((resolve) => setTimeout(resolve, 3000)) - return streamChat(context)(messages, { - onStreamedMessage, - isRetrying: true, - }) + if (!res.body) { + throw new Error('The response body is empty.') } - if (dataString.includes('[DONE]')) break - if (dataString.includes('"error":')) { - return { error: JSON.parse(dataString).error } - } - parser.feed(dataString) - } - return { message } + let message = '' + + const reader = res.body.getReader() + const decoder = new TextDecoder() + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read() + if (done) { + break + } + const chunk = decoder.decode(value) + if (onStreamedMessage) onStreamedMessage(chunk) + message += chunk + if (abortController === null) { + reader.cancel() + break + } + } + + abortController = null + + return { message } + } catch (err) { + console.error(err) + // Ignore abort errors as they are expected. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((err as any).name === 'AbortError') { + abortController = null + return { error: { message: 'Request aborted' } } + } + + if (err instanceof Error) return { error: { message: err.message } } + + return { error: { message: 'Failed to fetch the chat response.' } } + } } diff --git a/packages/embeds/react/package.json b/packages/embeds/react/package.json index 32c2ad42f..2f2adba7e 100644 --- a/packages/embeds/react/package.json +++ b/packages/embeds/react/package.json @@ -1,6 +1,6 @@ { "name": "@typebot.io/react", - "version": "0.0.64", + "version": "0.0.65", "description": "React library to display typebots on your website", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 778d7a2b0..0df34706d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -517,6 +517,9 @@ importers: '@typebot.io/react': specifier: workspace:* version: link:../../packages/embeds/react + ai: + specifier: ^2.1.3 + version: 2.1.3(react@18.2.0)(svelte@3.59.1)(vue@3.3.4) aws-sdk: specifier: 2.1384.0 version: 2.1384.0 @@ -550,6 +553,9 @@ importers: openai: specifier: 3.2.1 version: 3.2.1 + openai-edge: + specifier: ^1.1.0 + version: 1.1.0 qs: specifier: 6.11.2 version: 6.11.2 @@ -9034,6 +9040,89 @@ packages: - supports-color dev: false + /@vue/compiler-core@3.3.4: + resolution: {integrity: sha512-cquyDNvZ6jTbf/+x+AgM2Arrp6G4Dzbb0R64jiG804HRMfRiFXWI6kqUVqZ6ZR0bQhIoQjB4+2bhNtVwndW15g==} + dependencies: + '@babel/parser': 7.22.3 + '@vue/shared': 3.3.4 + estree-walker: 2.0.2 + source-map-js: 1.0.2 + dev: false + + /@vue/compiler-dom@3.3.4: + resolution: {integrity: sha512-wyM+OjOVpuUukIq6p5+nwHYtj9cFroz9cwkfmP9O1nzH68BenTTv0u7/ndggT8cIQlnBeOo6sUT/gvHcIkLA5w==} + dependencies: + '@vue/compiler-core': 3.3.4 + '@vue/shared': 3.3.4 + dev: false + + /@vue/compiler-sfc@3.3.4: + resolution: {integrity: sha512-6y/d8uw+5TkCuzBkgLS0v3lSM3hJDntFEiUORM11pQ/hKvkhSKZrXW6i69UyXlJQisJxuUEJKAWEqWbWsLeNKQ==} + dependencies: + '@babel/parser': 7.22.3 + '@vue/compiler-core': 3.3.4 + '@vue/compiler-dom': 3.3.4 + '@vue/compiler-ssr': 3.3.4 + '@vue/reactivity-transform': 3.3.4 + '@vue/shared': 3.3.4 + estree-walker: 2.0.2 + magic-string: 0.30.0 + postcss: 8.4.23 + source-map-js: 1.0.2 + dev: false + + /@vue/compiler-ssr@3.3.4: + resolution: {integrity: sha512-m0v6oKpup2nMSehwA6Uuu+j+wEwcy7QmwMkVNVfrV9P2qE5KshC6RwOCq8fjGS/Eak/uNb8AaWekfiXxbBB6gQ==} + dependencies: + '@vue/compiler-dom': 3.3.4 + '@vue/shared': 3.3.4 + dev: false + + /@vue/reactivity-transform@3.3.4: + resolution: {integrity: sha512-MXgwjako4nu5WFLAjpBnCj/ieqcjE2aJBINUNQzkZQfzIZA4xn+0fV1tIYBJvvva3N3OvKGofRLvQIwEQPpaXw==} + dependencies: + '@babel/parser': 7.22.3 + '@vue/compiler-core': 3.3.4 + '@vue/shared': 3.3.4 + estree-walker: 2.0.2 + magic-string: 0.30.0 + dev: false + + /@vue/reactivity@3.3.4: + resolution: {integrity: sha512-kLTDLwd0B1jG08NBF3R5rqULtv/f8x3rOFByTDz4J53ttIQEDmALqKqXY0J+XQeN0aV2FBxY8nJDf88yvOPAqQ==} + dependencies: + '@vue/shared': 3.3.4 + dev: false + + /@vue/runtime-core@3.3.4: + resolution: {integrity: sha512-R+bqxMN6pWO7zGI4OMlmvePOdP2c93GsHFM/siJI7O2nxFRzj55pLwkpCedEY+bTMgp5miZ8CxfIZo3S+gFqvA==} + dependencies: + '@vue/reactivity': 3.3.4 + '@vue/shared': 3.3.4 + dev: false + + /@vue/runtime-dom@3.3.4: + resolution: {integrity: sha512-Aj5bTJ3u5sFsUckRghsNjVTtxZQ1OyMWCr5dZRAPijF/0Vy4xEoRCwLyHXcj4D0UFbJ4lbx3gPTgg06K/GnPnQ==} + dependencies: + '@vue/runtime-core': 3.3.4 + '@vue/shared': 3.3.4 + csstype: 3.1.2 + dev: false + + /@vue/server-renderer@3.3.4(vue@3.3.4): + resolution: {integrity: sha512-Q6jDDzR23ViIb67v+vM1Dqntu+HUexQcsWKhhQa4ARVzxOY2HbC7QRW/ggkDBd5BU+uM1sV6XOAP0b216o34JQ==} + peerDependencies: + vue: 3.3.4 + dependencies: + '@vue/compiler-ssr': 3.3.4 + '@vue/shared': 3.3.4 + vue: 3.3.4 + dev: false + + /@vue/shared@3.3.4: + resolution: {integrity: sha512-7OjdcV8vQ74eiz1TZLzZP4JwqM5fA94K6yntPS5Z25r9HDuGNzaGdgvwKYq6S+MxwF0TFRwe50fIR/MYnakdkQ==} + dev: false + /@webassemblyjs/ast@1.11.1: resolution: {integrity: sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==} dependencies: @@ -9241,6 +9330,31 @@ packages: indent-string: 4.0.0 dev: false + /ai@2.1.3(react@18.2.0)(svelte@3.59.1)(vue@3.3.4): + resolution: {integrity: sha512-TwSSYf7YZdwTc9ZdV0qnvToesEePVvH0UlzYmLY44uOHkhNYYsPw5wh7Y+GwS/mHFBnPOm7NFDZbtFGGn+D0mQ==} + engines: {node: '>=14.6'} + peerDependencies: + react: ^18.0.0 + svelte: ^3.29.0 + vue: ^3.3.4 + peerDependenciesMeta: + react: + optional: true + svelte: + optional: true + vue: + optional: true + dependencies: + eventsource-parser: 1.0.0 + nanoid: 3.3.6 + react: 18.2.0 + sswr: 1.10.0(svelte@3.59.1) + svelte: 3.59.1 + swr: 2.1.5(react@18.2.0) + swrv: 1.0.3(vue@3.3.4) + vue: 3.3.4 + dev: false + /ajv-formats@2.1.1(ajv@8.12.0): resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -15508,6 +15622,13 @@ packages: '@jridgewell/sourcemap-codec': 1.4.15 dev: false + /magic-string@0.30.0: + resolution: {integrity: sha512-LA+31JYDJLs82r2ScLrlz1GjSgu66ZV518eyWT+S8VhyQn/JL0u9MeBOvQMGYiPk1DBiSN9DDMOcXvigJZaViQ==} + engines: {node: '>=12'} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + dev: false + /make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} @@ -16614,6 +16735,11 @@ packages: is-wsl: 2.2.0 dev: false + /openai-edge@1.1.0: + resolution: {integrity: sha512-VkXT7yhKjr1QlC/wIMuPW/s003cPHUtiNjt/kH+54iKDZZ7eiGMPuymhpSWJ/zVZfVz6+94okGd6rNuu7JRBDw==} + engines: {node: '>=12'} + dev: false + /openai@3.2.1: resolution: {integrity: sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==} dependencies: @@ -19451,6 +19577,15 @@ packages: number-is-nan: 1.0.1 dev: false + /sswr@1.10.0(svelte@3.59.1): + resolution: {integrity: sha512-nLWAJSQy3h8t7rrbTXanRyVHuQPj4PwKIVGe4IMlxJFdhyaxnN/JGACnvQKGDeWiTGYIZIx/jRuUsPEF0867Pg==} + peerDependencies: + svelte: ^3.29.0 + dependencies: + svelte: 3.59.1 + swrev: 3.0.0 + dev: false + /stable@0.1.8: resolution: {integrity: sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==} deprecated: 'Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility' @@ -19824,6 +19959,11 @@ packages: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} + /svelte@3.59.1: + resolution: {integrity: sha512-pKj8fEBmqf6mq3/NfrB9SLtcJcUvjYSWyePlfCqN9gujLB25RitWK8PvFzlwim6hD/We35KbPlRteuA6rnPGcQ==} + engines: {node: '>= 8'} + dev: false + /svg-parser@2.0.4: resolution: {integrity: sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==} dev: false @@ -19856,6 +19996,18 @@ packages: use-sync-external-store: 1.2.0(react@18.2.0) dev: false + /swrev@3.0.0: + resolution: {integrity: sha512-QJuZiptdOmbDY45pECBRVEgnoBlOKjeT2MWVz04wKHpWX15hM3P7EjcIbHDg5yLoPCMQ7to3349MEE+l9QF5HA==} + dev: false + + /swrv@1.0.3(vue@3.3.4): + resolution: {integrity: sha512-sl+eLEE+aPPjhP1E8gQ75q3RPRyw5Gd/kROnrTFo3+LkCeLskv7F+uAl5W97wgJkzitobL6FLsRPVm0DgIgN8A==} + peerDependencies: + vue: '>=3.2.26 < 4' + dependencies: + vue: 3.3.4 + dev: false + /symbol-observable@1.0.1: resolution: {integrity: sha512-Kb3PrPYz4HanVF1LVGuAdW6LoVgIwjUYJGzFe7NDrBLCN4lsV/5J0MFurV+ygS4bRVwrCEt2c7MQ1R2a72oJDw==} engines: {node: '>=0.10.0'} @@ -20992,6 +21144,16 @@ packages: fsevents: 2.3.2 dev: false + /vue@3.3.4: + resolution: {integrity: sha512-VTyEYn3yvIeY1Py0WaYGZsXnz3y5UnGi62GjVEqvEGPl6nxbOrCXbVOTQWBEJUqAyTUk2uJ5JLVnYJ6ZzGbrSw==} + dependencies: + '@vue/compiler-dom': 3.3.4 + '@vue/compiler-sfc': 3.3.4 + '@vue/runtime-dom': 3.3.4 + '@vue/server-renderer': 3.3.4(vue@3.3.4) + '@vue/shared': 3.3.4 + dev: false + /w3c-keyname@2.2.7: resolution: {integrity: sha512-XB8aa62d4rrVfoZYQaYNy3fy+z4nrfy2ooea3/0BnBzXW0tSdZ+lRgjzBZhk0La0H6h8fVyYCxx/qkQcAIuvfg==} dev: false