blsmedia/bot
2
0
Fork 0
Code Activity

⚡ (openai) Use Vercel's AI SDK for streaming

Browse Source
This commit is contained in:
Baptiste Arnaud
2023-06-20 17:23:14 +02:00
parent 7c2e5740dc
commit 3be39cbc78
7 changed files with 254 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/viewer/package.json
View File

@ -19,6 +19,7 @@
"@typebot.io/js": "workspace:*",
"@typebot.io/prisma": "workspace:*",
"@typebot.io/react": "workspace:*",
"ai": "^2.1.3",
"aws-sdk": "2.1384.0",
"bot-engine": "workspace:*",
"cors": "2.8.5",
@ -30,6 +31,7 @@
"nextjs-cors": "2.1.2",
"nodemailer": "6.9.2",
"openai": "3.2.1",
"openai-edge": "^1.1.0",
"qs": "6.11.2",
"react": "18.2.0",
"react-dom": "18.2.0",

33
apps/viewer/src/features/blocks/integrations/openai/getChatCompletionStream.ts
View File

@ -6,10 +6,12 @@ import {
OpenAICredentials,
} from '@typebot.io/schemas/features/blocks/integrations/openai'
import { SessionState } from '@typebot.io/schemas/features/chat'
import type {
import { OpenAIStream } from 'ai'
import {
ChatCompletionRequestMessage,
CreateChatCompletionRequest,
} from 'openai'
Configuration,
OpenAIApi,
} from 'openai-edge'
export const getChatCompletionStream =
(conn: Connection) =>
@ -37,19 +39,18 @@ export const getChatCompletionStream =
options.advancedSettings?.temperature
)
const res = await fetch('https://api.openai.com/v1/chat/completions', {
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${apiKey}`,
},
method: 'POST',
body: JSON.stringify({
messages,
model: options.model,
temperature,
stream: true,
} satisfies CreateChatCompletionRequest),
const config = new Configuration({
apiKey,
})
return res.body
const openai = new OpenAIApi(config)
const response = await openai.createChatCompletion({
model: options.model,
temperature,
stream: true,
messages,
})
return OpenAIStream(response)
}

10
apps/viewer/src/pages/api/integrations/openai/streamer.ts
View File

@ -1,6 +1,7 @@
import { getChatCompletionStream } from '@/features/blocks/integrations/openai/getChatCompletionStream'
import { connect } from '@planetscale/database'
import { IntegrationBlockType, SessionState } from '@typebot.io/schemas'
import { StreamingTextResponse } from 'ai'
import { ChatCompletionRequestMessage } from 'openai'
export const config = {
@ -15,8 +16,7 @@ const handler = async (req: Request) => {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'POST',
'Access-Control-Expose-Headers': 'Content-Length, X-JSON',
'Access-Control-Allow-Headers':
'apikey,X-Client-Info, Content-Type, Authorization, Accept, Accept-Language, X-Authorization',
'Access-Control-Allow-Headers': '*',
},
})
}
@ -66,12 +66,10 @@ const handler = async (req: Request) => {
messages
)
if (!stream) return new Response('Missing credentials', { status: 400 })
if (!stream) return new Response('Could not create stream', { status: 400 })
return new Response(stream, {
status: 200,
return new StreamingTextResponse(stream, {
headers: {
'Content-Type': 'application/json; charset=utf-8',
'Access-Control-Allow-Origin': '*',
},
})

2
packages/embeds/js/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@typebot.io/js",
"version": "0.0.64",
"version": "0.0.65",
"description": "Javascript library to display typebots on your website",
"type": "module",
"main": "dist/index.js",

119
packages/embeds/js/src/features/blocks/integrations/openai/streamChat.ts
View File

@ -1,10 +1,8 @@
import { getOpenAiStreamerQuery } from '@/queries/getOpenAiStreamerQuery'
import { ClientSideActionContext } from '@/types'
import {
createParser,
ParsedEvent,
ReconnectInterval,
} from 'eventsource-parser'
import { guessApiHost } from '@/utils/guessApiHost'
import { isNotEmpty } from '@typebot.io/lib/utils'
let abortController: AbortController | null = null
export const streamChat =
(context: ClientSideActionContext) =>
@ -13,59 +11,76 @@ export const streamChat =
content?: string | undefined
role?: 'system' | 'user' | 'assistant' | undefined
}[],
{
onStreamedMessage,
isRetrying,
}: { onStreamedMessage?: (message: string) => void; isRetrying?: boolean }
{ onStreamedMessage }: { onStreamedMessage?: (message: string) => void }
): Promise<{ message?: string; error?: object }> => {
const data = await getOpenAiStreamerQuery(context)(messages)
try {
abortController = new AbortController()
if (!data) return { error: { message: "Couldn't get streamer data" } }
const apiHost = context.apiHost
let message = ''
const res = await fetch(
`${
isNotEmpty(apiHost) ? apiHost : guessApiHost()
}/api/integrations/openai/streamer`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
sessionId: context.sessionId,
messages,
}),
signal: abortController.signal,
}
)
const reader = data.getReader()
const decoder = new TextDecoder()
const onParse = (event: ParsedEvent | ReconnectInterval) => {
if (event.type === 'event') {
const data = event.data
try {
const json = JSON.parse(data) as {
choices: { delta: { content: string } }[]
}
const text = json.choices.at(0)?.delta.content
if (!text) return
message += text
onStreamedMessage?.(message)
} catch (e) {
console.error(e)
if (!res.ok) {
return {
error: {
message: (await res.text()) || 'Failed to fetch the chat response.',
},
}
}
}
const parser = createParser(onParse)
// eslint-disable-next-line no-constant-condition
while (true) {
const { value, done } = await reader.read()
if (done || !value) break
const dataString = decoder.decode(value)
if (dataString.includes('503 Service Temporarily Unavailable')) {
if (isRetrying)
return { error: { message: "Couldn't get streamer data" } }
await new Promise((resolve) => setTimeout(resolve, 3000))
return streamChat(context)(messages, {
onStreamedMessage,
isRetrying: true,
})
if (!res.body) {
throw new Error('The response body is empty.')
}
if (dataString.includes('[DONE]')) break
if (dataString.includes('"error":')) {
return { error: JSON.parse(dataString).error }
}
parser.feed(dataString)
}
return { message }
let message = ''
const reader = res.body.getReader()
const decoder = new TextDecoder()
// eslint-disable-next-line no-constant-condition
while (true) {
const { done, value } = await reader.read()
if (done) {
break
}
const chunk = decoder.decode(value)
if (onStreamedMessage) onStreamedMessage(chunk)
message += chunk
if (abortController === null) {
reader.cancel()
break
}
}
abortController = null
return { message }
} catch (err) {
console.error(err)
// Ignore abort errors as they are expected.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if ((err as any).name === 'AbortError') {
abortController = null
return { error: { message: 'Request aborted' } }
}
if (err instanceof Error) return { error: { message: err.message } }
return { error: { message: 'Failed to fetch the chat response.' } }
}
}

2
packages/embeds/react/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@typebot.io/react",
"version": "0.0.64",
"version": "0.0.65",
"description": "React library to display typebots on your website",
"main": "dist/index.js",
"types": "dist/index.d.ts",

162
pnpm-lock.yaml generated
View File

@ -517,6 +517,9 @@ importers:
'@typebot.io/react':
specifier: workspace:*
version: link:../../packages/embeds/react
ai:
specifier: ^2.1.3
version: 2.1.3(react@18.2.0)(svelte@3.59.1)(vue@3.3.4)
aws-sdk:
specifier: 2.1384.0
version: 2.1384.0
@ -550,6 +553,9 @@ importers:
openai:
specifier: 3.2.1
version: 3.2.1
openai-edge:
specifier: ^1.1.0
version: 1.1.0
qs:
specifier: 6.11.2
version: 6.11.2
@ -9034,6 +9040,89 @@ packages:
- supports-color
dev: false
/@vue/compiler-core@3.3.4:
resolution: {integrity: sha512-cquyDNvZ6jTbf/+x+AgM2Arrp6G4Dzbb0R64jiG804HRMfRiFXWI6kqUVqZ6ZR0bQhIoQjB4+2bhNtVwndW15g==}
dependencies:
'@babel/parser': 7.22.3
'@vue/shared': 3.3.4
estree-walker: 2.0.2
source-map-js: 1.0.2
dev: false
/@vue/compiler-dom@3.3.4:
resolution: {integrity: sha512-wyM+OjOVpuUukIq6p5+nwHYtj9cFroz9cwkfmP9O1nzH68BenTTv0u7/ndggT8cIQlnBeOo6sUT/gvHcIkLA5w==}
dependencies:
'@vue/compiler-core': 3.3.4
'@vue/shared': 3.3.4
dev: false
/@vue/compiler-sfc@3.3.4:
resolution: {integrity: sha512-6y/d8uw+5TkCuzBkgLS0v3lSM3hJDntFEiUORM11pQ/hKvkhSKZrXW6i69UyXlJQisJxuUEJKAWEqWbWsLeNKQ==}
dependencies:
'@babel/parser': 7.22.3
'@vue/compiler-core': 3.3.4
'@vue/compiler-dom': 3.3.4
'@vue/compiler-ssr': 3.3.4
'@vue/reactivity-transform': 3.3.4
'@vue/shared': 3.3.4
estree-walker: 2.0.2
magic-string: 0.30.0
postcss: 8.4.23
source-map-js: 1.0.2
dev: false
/@vue/compiler-ssr@3.3.4:
resolution: {integrity: sha512-m0v6oKpup2nMSehwA6Uuu+j+wEwcy7QmwMkVNVfrV9P2qE5KshC6RwOCq8fjGS/Eak/uNb8AaWekfiXxbBB6gQ==}
dependencies:
'@vue/compiler-dom': 3.3.4
'@vue/shared': 3.3.4
dev: false
/@vue/reactivity-transform@3.3.4:
resolution: {integrity: sha512-MXgwjako4nu5WFLAjpBnCj/ieqcjE2aJBINUNQzkZQfzIZA4xn+0fV1tIYBJvvva3N3OvKGofRLvQIwEQPpaXw==}
dependencies:
'@babel/parser': 7.22.3
'@vue/compiler-core': 3.3.4
'@vue/shared': 3.3.4
estree-walker: 2.0.2
magic-string: 0.30.0
dev: false
/@vue/reactivity@3.3.4:
resolution: {integrity: sha512-kLTDLwd0B1jG08NBF3R5rqULtv/f8x3rOFByTDz4J53ttIQEDmALqKqXY0J+XQeN0aV2FBxY8nJDf88yvOPAqQ==}
dependencies:
'@vue/shared': 3.3.4
dev: false
/@vue/runtime-core@3.3.4:
resolution: {integrity: sha512-R+bqxMN6pWO7zGI4OMlmvePOdP2c93GsHFM/siJI7O2nxFRzj55pLwkpCedEY+bTMgp5miZ8CxfIZo3S+gFqvA==}
dependencies:
'@vue/reactivity': 3.3.4
'@vue/shared': 3.3.4
dev: false
/@vue/runtime-dom@3.3.4:
resolution: {integrity: sha512-Aj5bTJ3u5sFsUckRghsNjVTtxZQ1OyMWCr5dZRAPijF/0Vy4xEoRCwLyHXcj4D0UFbJ4lbx3gPTgg06K/GnPnQ==}
dependencies:
'@vue/runtime-core': 3.3.4
'@vue/shared': 3.3.4
csstype: 3.1.2
dev: false
/@vue/server-renderer@3.3.4(vue@3.3.4):
resolution: {integrity: sha512-Q6jDDzR23ViIb67v+vM1Dqntu+HUexQcsWKhhQa4ARVzxOY2HbC7QRW/ggkDBd5BU+uM1sV6XOAP0b216o34JQ==}
peerDependencies:
vue: 3.3.4
dependencies:
'@vue/compiler-ssr': 3.3.4
'@vue/shared': 3.3.4
vue: 3.3.4
dev: false
/@vue/shared@3.3.4:
resolution: {integrity: sha512-7OjdcV8vQ74eiz1TZLzZP4JwqM5fA94K6yntPS5Z25r9HDuGNzaGdgvwKYq6S+MxwF0TFRwe50fIR/MYnakdkQ==}
dev: false
/@webassemblyjs/ast@1.11.1:
resolution: {integrity: sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==}
dependencies:
@ -9241,6 +9330,31 @@ packages:
indent-string: 4.0.0
dev: false
/ai@2.1.3(react@18.2.0)(svelte@3.59.1)(vue@3.3.4):
resolution: {integrity: sha512-TwSSYf7YZdwTc9ZdV0qnvToesEePVvH0UlzYmLY44uOHkhNYYsPw5wh7Y+GwS/mHFBnPOm7NFDZbtFGGn+D0mQ==}
engines: {node: '>=14.6'}
peerDependencies:
react: ^18.0.0
svelte: ^3.29.0
vue: ^3.3.4
peerDependenciesMeta:
react:
optional: true
svelte:
optional: true
vue:
optional: true
dependencies:
eventsource-parser: 1.0.0
nanoid: 3.3.6
react: 18.2.0
sswr: 1.10.0(svelte@3.59.1)
svelte: 3.59.1
swr: 2.1.5(react@18.2.0)
swrv: 1.0.3(vue@3.3.4)
vue: 3.3.4
dev: false
/ajv-formats@2.1.1(ajv@8.12.0):
resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==}
peerDependencies:
@ -15508,6 +15622,13 @@ packages:
'@jridgewell/sourcemap-codec': 1.4.15
dev: false
/magic-string@0.30.0:
resolution: {integrity: sha512-LA+31JYDJLs82r2ScLrlz1GjSgu66ZV518eyWT+S8VhyQn/JL0u9MeBOvQMGYiPk1DBiSN9DDMOcXvigJZaViQ==}
engines: {node: '>=12'}
dependencies:
'@jridgewell/sourcemap-codec': 1.4.15
dev: false
/make-dir@3.1.0:
resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
engines: {node: '>=8'}
@ -16614,6 +16735,11 @@ packages:
is-wsl: 2.2.0
dev: false
/openai-edge@1.1.0:
resolution: {integrity: sha512-VkXT7yhKjr1QlC/wIMuPW/s003cPHUtiNjt/kH+54iKDZZ7eiGMPuymhpSWJ/zVZfVz6+94okGd6rNuu7JRBDw==}
engines: {node: '>=12'}
dev: false
/openai@3.2.1:
resolution: {integrity: sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==}
dependencies:
@ -19451,6 +19577,15 @@ packages:
number-is-nan: 1.0.1
dev: false
/sswr@1.10.0(svelte@3.59.1):
resolution: {integrity: sha512-nLWAJSQy3h8t7rrbTXanRyVHuQPj4PwKIVGe4IMlxJFdhyaxnN/JGACnvQKGDeWiTGYIZIx/jRuUsPEF0867Pg==}
peerDependencies:
svelte: ^3.29.0
dependencies:
svelte: 3.59.1
swrev: 3.0.0
dev: false
/stable@0.1.8:
resolution: {integrity: sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==}
deprecated: 'Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility'
@ -19824,6 +19959,11 @@ packages:
resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
engines: {node: '>= 0.4'}
/svelte@3.59.1:
resolution: {integrity: sha512-pKj8fEBmqf6mq3/NfrB9SLtcJcUvjYSWyePlfCqN9gujLB25RitWK8PvFzlwim6hD/We35KbPlRteuA6rnPGcQ==}
engines: {node: '>= 8'}
dev: false
/svg-parser@2.0.4:
resolution: {integrity: sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==}
dev: false
@ -19856,6 +19996,18 @@ packages:
use-sync-external-store: 1.2.0(react@18.2.0)
dev: false
/swrev@3.0.0:
resolution: {integrity: sha512-QJuZiptdOmbDY45pECBRVEgnoBlOKjeT2MWVz04wKHpWX15hM3P7EjcIbHDg5yLoPCMQ7to3349MEE+l9QF5HA==}
dev: false
/swrv@1.0.3(vue@3.3.4):
resolution: {integrity: sha512-sl+eLEE+aPPjhP1E8gQ75q3RPRyw5Gd/kROnrTFo3+LkCeLskv7F+uAl5W97wgJkzitobL6FLsRPVm0DgIgN8A==}
peerDependencies:
vue: '>=3.2.26 < 4'
dependencies:
vue: 3.3.4
dev: false
/symbol-observable@1.0.1:
resolution: {integrity: sha512-Kb3PrPYz4HanVF1LVGuAdW6LoVgIwjUYJGzFe7NDrBLCN4lsV/5J0MFurV+ygS4bRVwrCEt2c7MQ1R2a72oJDw==}
engines: {node: '>=0.10.0'}
@ -20992,6 +21144,16 @@ packages:
fsevents: 2.3.2
dev: false
/vue@3.3.4:
resolution: {integrity: sha512-VTyEYn3yvIeY1Py0WaYGZsXnz3y5UnGi62GjVEqvEGPl6nxbOrCXbVOTQWBEJUqAyTUk2uJ5JLVnYJ6ZzGbrSw==}
dependencies:
'@vue/compiler-dom': 3.3.4
'@vue/compiler-sfc': 3.3.4
'@vue/runtime-dom': 3.3.4
'@vue/server-renderer': 3.3.4(vue@3.3.4)
'@vue/shared': 3.3.4
dev: false
/w3c-keyname@2.2.7:
resolution: {integrity: sha512-XB8aa62d4rrVfoZYQaYNy3fy+z4nrfy2ooea3/0BnBzXW0tSdZ+lRgjzBZhk0La0H6h8fVyYCxx/qkQcAIuvfg==}
dev: false