blsmedia/bot
2
0
Fork 0
Code Activity

✨ (openai) Add create speech OpenAI action

Browse Source 
Closes #1025
This commit is contained in:
Baptiste Arnaud
2023-11-20 10:32:38 +01:00
parent f6ac3891f8
commit 1a44bf4935
23 changed files with 1772 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/builder/package.json
View File

@ -74,7 +74,7 @@
"nextjs-cors": "2.1.2", "nextjs-cors": "2.1.2",
"nodemailer": "6.9.3", "nodemailer": "6.9.3",
"nprogress": "0.2.0", "nprogress": "0.2.0",
"openai": "4.11.1", "openai": "4.19.0",
"papaparse": "5.4.1", "papaparse": "5.4.1",
"posthog-js": "1.77.1", "posthog-js": "1.77.1",
"posthog-node": "3.1.1", "posthog-node": "3.1.1",

206
apps/builder/public/templates/audio-chat-gpt.json Normal file
View File

@ -0,0 +1,206 @@
{
"version": "6",
"id": "clp6onbn200011ab379x5gnea",
"name": "Audio ChatGPT",
"icon": "🔈",
"events": [
{
"id": "ewnfbo0exlu7ihfu2lu2lusm",
"outgoingEdgeId": "knz1ln1so0dfyth76qjkjn1p",
"graphCoordinates": { "x": -228.25, "y": -123.31 },
"type": "start"
}
],
"groups": [
{
"id": "qfrz5nwm63g12dajsjxothb5",
"title": "User input",
"graphCoordinates": { "x": 105.17, "y": -56.29 },
"blocks": [
{
"id": "ovgk70u0kfxrbtz9dy4e040o",
"type": "text input",
"options": { "variableId": "vudksu3zyrat6s1bq6qne0rx3" }
},
{
"id": "m4jadtknjb3za3gvxj1xdn1k",
"outgoingEdgeId": "fpj0xacppqd1s5slyljzhzc9",
"type": "Set variable",
"options": {
"variableId": "vabkycu0qqff5d6ar2ama16pf",
"type": "Append value(s)",
"item": "{{User Message}}"
}
}
]
},
{
"id": "a6ymhjwtkqwp8t127plz8qmk",
"title": "ChatGPT reply",
"graphCoordinates": { "x": 445.12, "y": -56.2 },
"blocks": [
{
"id": "xikptnw1lp1qxdqo10qhmwy1",
"type": "OpenAI",
"options": {
"task": "Create chat completion",
"model": "gpt-3.5-turbo",
"messages": [
{
"id": "wsdxha9db58gk2v9n1j10m7c",
"role": "Dialogue",
"dialogueVariableId": "vabkycu0qqff5d6ar2ama16pf",
"startsBy": "user"
}
],
"responseMapping": [
{
"id": "p8ksqi2jhyzid2od3dikv299",
"valueToExtract": "Message content",
"variableId": "vni6kwbch8zlq92dclgcivzyr"
}
],
"credentialsId": "clp6ooc3700031ab30yof27jm"
}
},
{
"id": "yblc864bzipaqfja7b2o3oo0",
"type": "Set variable",
"options": {
"variableId": "vabkycu0qqff5d6ar2ama16pf",
"type": "Append value(s)",
"item": "{{Assistant Message}}"
}
},
{
"id": "prsimdxdol42ty2parzgx8am",
"type": "OpenAI",
"options": {
"credentialsId": "clp6ooc3700031ab30yof27jm",
"task": "Create speech",
"model": "tts-1",
"input": "{{Assistant Message}}",
"voice": "alloy",
"saveUrlInVariableId": "vgr0iwg95npp7pztkmdyn89m1"
}
},
{
"id": "f6onszqys2tx7pw3tshb2vcy",
"outgoingEdgeId": "dw9jclv60i040saiikj3a386",
"type": "audio",
"content": { "url": "{{Assistant audio message}}" }
}
]
},
{
"id": "c5f00f3oclwi1srcz10jjt9u",
"title": "Intro",
"graphCoordinates": { "x": -222.61, "y": -54.39 },
"blocks": [
{
"id": "dp5gx25j73fgmcj9582ydik9",
"type": "OpenAI",
"options": {
"credentialsId": "clp6ooc3700031ab30yof27jm",
"task": "Create speech",
"model": "tts-1",
"input": "Hi there! How can I help?",
"voice": "alloy",
"saveUrlInVariableId": "vxw4quja426402hvhtm33tsp3"
}
},
{
"id": "dmab8kc35uh84vvw1a53xbjn",
"outgoingEdgeId": "xnao10ucc1hbynv3pmk1t4by",
"type": "audio",
"content": { "url": "{{Welcome audio}}" }
}
]
},
{
"id": "yswu9fml4zflxaqlujb94ir8",
"title": "",
"graphCoordinates": { "x": 19.51, "y": -338.93 },
"blocks": [
{
"id": "okm2zz32zn8b60u1vkfrv9ca",
"type": "text",
"content": {
"richText": [
{
"type": "p",
"children": [
{
"text": "You need to add your OpenAI credentials to make this bot work. 🪄"
}
]
}
]
}
},
{
"id": "m5a1d0vhsrpyvvvyj89awxzc",
"type": "text",
"content": {
"richText": [
{
"type": "p",
"children": [
{
"text": "Once it's done, delete this group and connect the "
},
{ "text": "Start", "bold": true },
{ "text": " event with " },
{ "text": "Intro", "bold": true },
{ "text": " 🚀\n" }
]
}
]
}
}
]
}
],
"edges": [
{
"id": "fpj0xacppqd1s5slyljzhzc9",
"from": { "blockId": "m4jadtknjb3za3gvxj1xdn1k" },
"to": { "groupId": "a6ymhjwtkqwp8t127plz8qmk" }
},
{
"id": "xnao10ucc1hbynv3pmk1t4by",
"from": { "blockId": "dmab8kc35uh84vvw1a53xbjn" },
"to": { "groupId": "qfrz5nwm63g12dajsjxothb5" }
},
{
"id": "dw9jclv60i040saiikj3a386",
"from": { "blockId": "f6onszqys2tx7pw3tshb2vcy" },
"to": { "groupId": "qfrz5nwm63g12dajsjxothb5" }
},
{
"from": { "eventId": "ewnfbo0exlu7ihfu2lu2lusm" },
"to": { "groupId": "yswu9fml4zflxaqlujb94ir8" },
"id": "knz1ln1so0dfyth76qjkjn1p"
}
],
"variables": [
{ "id": "vni6kwbch8zlq92dclgcivzyr", "name": "Assistant Message" },
{ "id": "vudksu3zyrat6s1bq6qne0rx3", "name": "User Message" },
{ "id": "vabkycu0qqff5d6ar2ama16pf", "name": "Chat history" },
{ "id": "vxw4quja426402hvhtm33tsp3", "name": "Welcome audio" },
{ "id": "vgr0iwg95npp7pztkmdyn89m1", "name": "Assistant audio message" }
],
"theme": {},
"selectedThemeTemplateId": null,
"settings": {},
"createdAt": "2023-11-20T09:06:40.430Z",
"updatedAt": "2023-11-20T09:20:01.662Z",
"folderId": null,
"publicId": null,
"customDomain": null,
"workspaceId": "freeWorkspace",
"resultsTablePreferences": null,
"isArchived": false,
"isClosed": false,
"whatsAppCredentialsId": null
}

290
apps/builder/public/templates/openai-conditions.json Normal file
View File

@ -0,0 +1,290 @@
{
"version": "6",
"id": "clp6pe8dy00051ab3n6coxt62",
"name": "ChatGPT condition",
"events": [
{
"id": "ewnfbo0exlu7ihfu2lu2lusm",
"outgoingEdgeId": "gj1gs8hdembrsw84aafd1hbj",
"graphCoordinates": { "x": -228.25, "y": -123.31 },
"type": "start"
}
],
"groups": [
{
"id": "qfrz5nwm63g12dajsjxothb5",
"title": "User input",
"graphCoordinates": { "x": -107.01, "y": -53.23 },
"blocks": [
{
"id": "emjcjxlvzm2xex10exq4wf5h",
"type": "text",
"content": {
"richText": [
{ "type": "p", "children": [{ "text": "How can we help?" }] }
]
}
},
{
"id": "ovgk70u0kfxrbtz9dy4e040o",
"outgoingEdgeId": "n396v90ad7quz0gwygr6n5fc",
"type": "text input",
"options": { "variableId": "vudksu3zyrat6s1bq6qne0rx3" }
}
]
},
{
"id": "bh3uva3254p0jgp46gj92way",
"title": "Condition 🧠",
"graphCoordinates": { "x": 228.67, "y": -50.67 },
"blocks": [
{
"id": "wdg7upk4oqp602jqjn06gjf6",
"type": "OpenAI",
"options": {
"task": "Create chat completion",
"model": "gpt-4-1106-preview",
"messages": [
{
"id": "s7s7uaurqlmsn3r89c10mk98",
"role": "system",
"content": "You are helpful assistant doing customer support for a software called Typebot.\n\nIf the user is asking a question about his account, please say \"ACCOUNT\".\n\nIf the user wants to talk to a human, please say \"HUMAN\".\n\nOtherwise, say \"OK\""
},
{
"id": "zrgypmt1wlogakfl06gfxpgk",
"role": "user",
"content": "Can I talk to a human?"
},
{
"id": "i6ldg74yr9n185oumozb3r6b",
"role": "assistant",
"content": "HUMAN"
},
{
"id": "eoxa3dxtw8wjdyv9efnxryxk",
"role": "user",
"content": "I need to check my account"
},
{
"id": "nb7sy9x7g07w5s1sxb83v295",
"role": "assistant",
"content": "ACCOUNT"
},
{
"id": "zazen7p0cyawtix7der2e923",
"role": "user",
"content": "{{User Message}}"
}
],
"responseMapping": [
{
"id": "s7s7uaurqlmsn3r89c10mk98",
"valueToExtract": "Message content",
"variableId": "vni6kwbch8zlq92dclgcivzyr"
}
]
}
},
{
"id": "b63f0et5y70mragfcrmezifh",
"outgoingEdgeId": "wkecv7zg757mbsnbfdej6uph",
"type": "Condition",
"items": [
{
"id": "d0bax1yf18x2bj5zt38vr6xs",
"outgoingEdgeId": "optu30rn5vjiwueijh0utvoi",
"content": {
"comparisons": [
{
"id": "iiv6h6ssgusrfza4vped0iqe",
"variableId": "vni6kwbch8zlq92dclgcivzyr",
"comparisonOperator": "Contains",
"value": "ACCOUNT"
}
]
}
},
{
"id": "ulf15sjzk9b7df95rzaqe40j",
"outgoingEdgeId": "ft02qowy1n8uy2k5dzdo0j22",
"content": {
"comparisons": [
{
"id": "qekpq2av325h7rkmnrvtumxn",
"variableId": "vni6kwbch8zlq92dclgcivzyr",
"comparisonOperator": "Contains",
"value": "HUMAN"
}
]
}
}
]
}
]
},
{
"id": "flt2vtb4pb6mvlvyrrxonzwf",
"title": "Account",
"graphCoordinates": { "x": 585.48, "y": -214.13 },
"blocks": [
{
"id": "h45t9e5yxc2fpfiv49gvmwjj",
"type": "text",
"content": {
"richText": [
{
"type": "p",
"children": [
{ "text": "Ok feel free to check your account here" }
]
}
]
}
},
{
"id": "fn0s0ezabail96c5n0xs24aq",
"type": "choice input",
"items": [{ "id": "tslv59v8oqbfpnd21la8qnz4", "content": "Account" }]
},
{
"id": "r16g3avw7mwzi12srqisuex0",
"type": "Redirect",
"options": { "url": "https://google.com" }
}
]
},
{
"id": "jn9w80afa6a66czzjfto8tmt",
"title": "Human",
"graphCoordinates": { "x": 586.68, "y": 153.73 },
"blocks": [
{
"id": "nvkp4zm7f24b0g3b25u3rb59",
"type": "text",
"content": {
"richText": [
{ "type": "p", "children": [{ "text": "I'm a human" }] }
]
}
}
]
},
{
"id": "el01d0j1db9kp6v8wlrk9dob",
"title": "Else",
"graphCoordinates": { "x": 581.46, "y": 315.66 },
"blocks": [
{
"id": "tpy9wjrwmhw16xjgs8htao04",
"type": "text",
"content": {
"richText": [{ "type": "p", "children": [{ "text": "Else" }] }]
}
}
]
},
{
"id": "vafybpsjqcbrbbhi8pwl0gic",
"title": "",
"graphCoordinates": { "x": 6.52, "y": -346.17 },
"blocks": [
{
"id": "q1tc6z6xfl4jtrxdvv8phgil",
"type": "text",
"content": {
"richText": [
{
"type": "p",
"children": [
{
"text": "You need to add your OpenAI credentials to make this bot work. 🪄\n"
}
]
}
]
}
},
{
"id": "b3ahk7pzsh4abp3bkdlijc0v",
"type": "text",
"content": {
"richText": [
{
"type": "p",
"children": [
{
"text": "Once it's done, delete this group and connect the "
},
{ "text": "Start", "bold": true },
{ "text": " event with " },
{ "text": "Intro", "bold": true },
{ "text": " 🚀\n" }
]
}
]
}
}
]
}
],
"edges": [
{
"id": "h5sk58j0ryrxmfv4gmw7r4dw",
"from": { "blockId": "gphm5wy1md9cunwkdtbzg6nq" },
"to": { "groupId": "qfrz5nwm63g12dajsjxothb5" }
},
{
"id": "y8ml9ljnsydol9b42fd9zdve",
"from": { "blockId": "myldn1l1nfdwwm8qvza71rwv" },
"to": { "groupId": "qfrz5nwm63g12dajsjxothb5" }
},
{
"id": "optu30rn5vjiwueijh0utvoi",
"from": {
"blockId": "b63f0et5y70mragfcrmezifh",
"itemId": "d0bax1yf18x2bj5zt38vr6xs"
},
"to": { "groupId": "flt2vtb4pb6mvlvyrrxonzwf" }
},
{
"id": "ft02qowy1n8uy2k5dzdo0j22",
"from": {
"blockId": "b63f0et5y70mragfcrmezifh",
"itemId": "ulf15sjzk9b7df95rzaqe40j"
},
"to": { "groupId": "jn9w80afa6a66czzjfto8tmt" }
},
{
"id": "wkecv7zg757mbsnbfdej6uph",
"from": { "blockId": "b63f0et5y70mragfcrmezifh" },
"to": { "groupId": "el01d0j1db9kp6v8wlrk9dob" }
},
{
"id": "n396v90ad7quz0gwygr6n5fc",
"from": { "blockId": "ovgk70u0kfxrbtz9dy4e040o" },
"to": { "groupId": "bh3uva3254p0jgp46gj92way" }
},
{
"from": { "eventId": "ewnfbo0exlu7ihfu2lu2lusm" },
"to": { "groupId": "vafybpsjqcbrbbhi8pwl0gic" },
"id": "gj1gs8hdembrsw84aafd1hbj"
}
],
"variables": [
{ "id": "vni6kwbch8zlq92dclgcivzyr", "name": "Assistant Message" },
{ "id": "vudksu3zyrat6s1bq6qne0rx3", "name": "User Message" }
],
"theme": {},
"selectedThemeTemplateId": null,
"settings": {},
"createdAt": "2023-11-20T09:27:35.926Z",
"updatedAt": "2023-11-20T09:27:59.586Z",
"icon": "🧠",
"folderId": null,
"publicId": null,
"customDomain": null,
"resultsTablePreferences": null,
"isArchived": false,
"isClosed": false,
"whatsAppCredentialsId": null
}

4
apps/builder/src/features/blocks/integrations/openai/api/listModels.ts
View File

@ -25,6 +25,7 @@ export const listModels = authenticatedProcedure
workspaceId: z.string(), workspaceId: z.string(),
baseUrl: z.string(), baseUrl: z.string(),
apiVersion: z.string().optional(), apiVersion: z.string().optional(),
type: z.enum(['gpt', 'tts']),
}) })
) )
.output( .output(
@ -34,7 +35,7 @@ export const listModels = authenticatedProcedure
) )
.query( .query(
async ({ async ({
input: { credentialsId, workspaceId, baseUrl, apiVersion }, input: { credentialsId, workspaceId, baseUrl, apiVersion, type },
ctx: { user }, ctx: { user },
}) => { }) => {
const workspace = await prisma.workspace.findFirst({ const workspace = await prisma.workspace.findFirst({
@ -97,6 +98,7 @@ export const listModels = authenticatedProcedure
return { return {
models: models:
models.data models.data
.filter((model) => model.id.includes(type))
.sort((a, b) => b.created - a.created) .sort((a, b) => b.created - a.created)
.map((model) => model.id) ?? [], .map((model) => model.id) ?? [],
} }

3
apps/builder/src/features/blocks/integrations/openai/components/createChatCompletion/ModelsDropdown.tsx → apps/builder/src/features/blocks/integrations/openai/components/ModelsDropdown.tsx
View File

@ -9,6 +9,7 @@ type Props = {
apiVersion?: string apiVersion?: string
credentialsId: string credentialsId: string
defaultValue?: string defaultValue?: string
type: 'gpt' | 'tts'
onChange: (model: string | undefined) => void onChange: (model: string | undefined) => void
} }
@ -18,6 +19,7 @@ export const ModelsDropdown = ({
defaultValue, defaultValue,
onChange, onChange,
credentialsId, credentialsId,
type,
}: Props) => { }: Props) => {
const { workspace } = useWorkspace() const { workspace } = useWorkspace()
const { showToast } = useToast() const { showToast } = useToast()
@ -28,6 +30,7 @@ export const ModelsDropdown = ({
baseUrl: baseUrl ?? defaultOpenAIOptions.baseUrl, baseUrl: baseUrl ?? defaultOpenAIOptions.baseUrl,
workspaceId: workspace?.id as string, workspaceId: workspace?.id as string,
apiVersion, apiVersion,
type,
}, },
{ {
enabled: !!workspace, enabled: !!workspace,

30
apps/builder/src/features/blocks/integrations/openai/components/OpenAINodeBody.tsx
View File

@ -1,29 +1,24 @@
import { SetVariableLabel } from '@/components/SetVariableLabel' import { SetVariableLabel } from '@/components/SetVariableLabel'
import { useTypebot } from '@/features/editor/providers/TypebotProvider' import { useTypebot } from '@/features/editor/providers/TypebotProvider'
import { Stack, Text } from '@chakra-ui/react' import { Stack, Text } from '@chakra-ui/react'
import { import { OpenAIBlock } from '@typebot.io/schemas/features/blocks/integrations/openai'
ChatCompletionOpenAIOptions,
CreateImageOpenAIOptions,
OpenAIBlock,
} from '@typebot.io/schemas/features/blocks/integrations/openai'
type Props = { type Props = {
task: NonNullable<OpenAIBlock['options']>['task'] options: OpenAIBlock['options']
responseMapping:
| ChatCompletionOpenAIOptions['responseMapping']
| CreateImageOpenAIOptions['responseMapping']
} }
export const OpenAINodeBody = ({ task, responseMapping }: Props) => { export const OpenAINodeBody = ({ options }: Props) => {
const { typebot } = useTypebot() const { typebot } = useTypebot()
return ( return (
<Stack> <Stack>
<Text color={task ? 'currentcolor' : 'gray.500'} noOfLines={1}> <Text color={options?.task ? 'currentcolor' : 'gray.500'} noOfLines={1}>
{task ?? 'Configure...'} {options?.task ?? 'Configure...'}
</Text> </Text>
{typebot && {typebot &&
responseMapping options &&
'responseMapping' in options &&
options.responseMapping
?.map((mapping) => mapping.variableId) ?.map((mapping) => mapping.variableId)
.map((variableId, idx) => .map((variableId, idx) =>
variableId ? ( variableId ? (
@ -34,6 +29,15 @@ export const OpenAINodeBody = ({ task, responseMapping }: Props) => {
/> />
) : null ) : null
)} )}
{typebot &&
options &&
'saveUrlInVariableId' in options &&
options.saveUrlInVariableId && (
<SetVariableLabel
variables={typebot.variables}
variableId={options.saveUrlInVariableId}
/>
)}
</Stack> </Stack>
) )
} }

30
apps/builder/src/features/blocks/integrations/openai/components/OpenAISettings.tsx
View File

@ -13,6 +13,7 @@ import { CredentialsDropdown } from '@/features/credentials/components/Credentia
import { import {
ChatCompletionOpenAIOptions, ChatCompletionOpenAIOptions,
CreateImageOpenAIOptions, CreateImageOpenAIOptions,
CreateSpeechOpenAIOptions,
OpenAIBlock, OpenAIBlock,
} from '@typebot.io/schemas/features/blocks/integrations/openai' } from '@typebot.io/schemas/features/blocks/integrations/openai'
import { OpenAICredentialsModal } from './OpenAICredentialsModal' import { OpenAICredentialsModal } from './OpenAICredentialsModal'
@ -24,6 +25,7 @@ import {
defaultOpenAIOptions, defaultOpenAIOptions,
openAITasks, openAITasks,
} from '@typebot.io/schemas/features/blocks/integrations/openai/constants' } from '@typebot.io/schemas/features/blocks/integrations/openai/constants'
import { OpenAICreateSpeechSettings } from './audio/OpenAICreateSpeechSettings'
type OpenAITask = (typeof openAITasks)[number] type OpenAITask = (typeof openAITasks)[number]
@ -47,15 +49,10 @@ export const OpenAISettings = ({
} }
const updateTask = (task: OpenAITask) => { const updateTask = (task: OpenAITask) => {
switch (task) { onOptionsChange({
case 'Create chat completion': { credentialsId: options?.credentialsId,
onOptionsChange({ task,
credentialsId: options?.credentialsId, } as OpenAIBlock['options'])
task,
})
break
}
}
} }
const updateBaseUrl = (baseUrl: string) => { const updateBaseUrl = (baseUrl: string) => {
@ -142,9 +139,12 @@ const OpenAITaskSettings = ({
options, options,
onOptionsChange, onOptionsChange,
}: { }: {
options: ChatCompletionOpenAIOptions | CreateImageOpenAIOptions options:
| ChatCompletionOpenAIOptions
| CreateImageOpenAIOptions
| CreateSpeechOpenAIOptions
onOptionsChange: (options: OpenAIBlock['options']) => void onOptionsChange: (options: OpenAIBlock['options']) => void
}) => { }): JSX.Element | null => {
switch (options.task) { switch (options.task) {
case 'Create chat completion': { case 'Create chat completion': {
return ( return (
@ -154,6 +154,14 @@ const OpenAITaskSettings = ({
/> />
) )
} }
case 'Create speech': {
return (
<OpenAICreateSpeechSettings
options={options}
onOptionsChange={onOptionsChange}
/>
)
}
case 'Create image': { case 'Create image': {
return null return null
} }

98
apps/builder/src/features/blocks/integrations/openai/components/audio/OpenAICreateSpeechSettings.tsx Normal file
View File

@ -0,0 +1,98 @@
import { CreateSpeechOpenAIOptions } from '@typebot.io/schemas/features/blocks/integrations/openai'
import { FormControl, FormLabel, Stack, Text } from '@chakra-ui/react'
import { TextLink } from '@/components/TextLink'
import { ModelsDropdown } from '../ModelsDropdown'
import { Textarea } from '@/components/inputs'
import { DropdownList } from '@/components/DropdownList'
import { openAIVoices } from '@typebot.io/schemas/features/blocks/integrations/openai/constants'
import { VariableSearchInput } from '@/components/inputs/VariableSearchInput'
import { Variable } from '@typebot.io/schemas'
const apiReferenceUrl =
'https://platform.openai.com/docs/api-reference/audio/createSpeech'
type Props = {
options: CreateSpeechOpenAIOptions
onOptionsChange: (options: CreateSpeechOpenAIOptions) => void
}
export const OpenAICreateSpeechSettings = ({
options,
onOptionsChange,
}: Props) => {
const updateModel = (model: string | undefined) => {
onOptionsChange({
...options,
model,
})
}
const updateInput = (input: string | undefined) => {
onOptionsChange({
...options,
input,
})
}
const updateVoice = (voice: (typeof openAIVoices)[number]) => {
onOptionsChange({
...options,
voice,
})
}
const updateSaveUrlInVariableId = (
variable: Pick<Variable, 'id' | 'name'> | undefined
) => {
onOptionsChange({
...options,
saveUrlInVariableId: variable?.id,
})
}
return (
<Stack spacing={4} pt="2">
<Text fontSize="sm" color="gray.500">
Read the{' '}
<TextLink href={apiReferenceUrl} isExternal>
API reference
</TextLink>{' '}
to better understand the available options.
</Text>
{options.credentialsId && (
<>
<ModelsDropdown
credentialsId={options.credentialsId}
defaultValue={options.model}
baseUrl={options.baseUrl}
apiVersion={options.apiVersion}
type="tts"
onChange={updateModel}
/>
<Textarea
defaultValue={options.input}
onChange={updateInput}
label="Input:"
/>
<FormControl>
<FormLabel>Voice:</FormLabel>
<DropdownList
currentItem={options.voice}
onItemSelect={updateVoice}
items={openAIVoices}
placeholder="Select a voice"
w="full"
/>
</FormControl>
<FormControl>
<FormLabel>Save URL:</FormLabel>
<VariableSearchInput
initialVariableId={options.saveUrlInVariableId}
onSelectVariable={updateSaveUrlInVariableId}
/>
</FormControl>
</>
)}
</Stack>
)
}

3
apps/builder/src/features/blocks/integrations/openai/components/createChatCompletion/OpenAIChatCompletionSettings.tsx
View File

@ -13,7 +13,7 @@ import {
import { TextLink } from '@/components/TextLink' import { TextLink } from '@/components/TextLink'
import { ChatCompletionResponseItem } from './ChatCompletionResponseItem' import { ChatCompletionResponseItem } from './ChatCompletionResponseItem'
import { NumberInput } from '@/components/inputs' import { NumberInput } from '@/components/inputs'
import { ModelsDropdown } from './ModelsDropdown' import { ModelsDropdown } from '../ModelsDropdown'
const apiReferenceUrl = const apiReferenceUrl =
'https://platform.openai.com/docs/api-reference/chat/create' 'https://platform.openai.com/docs/api-reference/chat/create'
@ -79,6 +79,7 @@ export const OpenAIChatCompletionSettings = ({
defaultValue={options.model} defaultValue={options.model}
baseUrl={options.baseUrl} baseUrl={options.baseUrl}
apiVersion={options.apiVersion} apiVersion={options.apiVersion}
type="gpt"
onChange={updateModel} onChange={updateModel}
/> />
<Accordion allowMultiple> <Accordion allowMultiple>

11
apps/builder/src/features/graph/components/nodes/block/BlockNodeContent.tsx
View File

@ -145,16 +145,7 @@ export const BlockNodeContent = ({
return <ChatwootNodeBody block={block} /> return <ChatwootNodeBody block={block} />
} }
case IntegrationBlockType.OPEN_AI: { case IntegrationBlockType.OPEN_AI: {
return ( return <OpenAINodeBody options={block.options} />
<OpenAINodeBody
task={block.options?.task}
responseMapping={
block.options && 'responseMapping' in block.options
? block.options.responseMapping
: []
}
/>
)
} }
case IntegrationBlockType.PIXEL: { case IntegrationBlockType.PIXEL: {
return <PixelNodeBody options={block.options} /> return <PixelNodeBody options={block.options} />

16
apps/builder/src/features/templates/data.ts
View File

@ -94,6 +94,14 @@ export const templates: TemplateProps[] = [
description: description:
'A bot that uses the ChatGPT model to generate responses based on the user input', 'A bot that uses the ChatGPT model to generate responses based on the user input',
}, },
{
name: 'Audio ChatGPT',
emoji: '🤖',
fileName: 'audio-chat-gpt.json',
description:
'An audio AI bot that uses the OpenAI block to generate responses based on the user input',
isNew: true,
},
{ {
name: 'ChatGPT personas', name: 'ChatGPT personas',
emoji: '🎭', emoji: '🎭',
@ -117,4 +125,12 @@ export const templates: TemplateProps[] = [
description: description:
'You are a dog insurance company. This bot allows you to collect information about the dog and provide a quote.', 'You are a dog insurance company. This bot allows you to collect information about the dog and provide a quote.',
}, },
{
name: 'OpenAI conditions',
emoji: '🧠',
fileName: 'openai-conditions.json',
isNew: true,
description:
'This is an example of how you can use the OpenAI block to take smart decisions based on the user input and redirect the conversation to the right path.',
},
] ]

31
apps/docs/docs/editor/blocks/integrations/openai.mdx
View File

@ -1,7 +1,7 @@
import { YoutubeEmbed } from '../../../../src/js/YoutubeEmbed.jsx'
# OpenAI # OpenAI
## Create chat completion
With the OpenAI block, you can create a chat completion based on your user queries and display the answer back to your typebot. With the OpenAI block, you can create a chat completion based on your user queries and display the answer back to your typebot.
<img <img
@ -25,6 +25,33 @@ Then you can give the OpenAI block access to this sequence of messages:
alt="OpenAI messages sequence" alt="OpenAI messages sequence"
/> />
## Create speech
This action allows you to transform a text input into an audio URL that you can reuse in your bot.
<div
style={{
position: 'relative',
paddingBottom: '64.5933014354067%',
height: 0,
}}
>
<iframe
src="https://www.loom.com/embed/ccca6cbf16ed4d01b513836775db06a3?sid=22a54baa-000b-435a-b770-bfaf32bfe453"
frameBorder={0}
webkitallowfullscreen
mozallowfullscreen
allowFullScreen
style={{
position: 'absolute',
top: 0,
left: 0,
width: '100%',
height: '100%',
}}
/>
</div>
## Using Multiple Open AI Blocks: Tips and Tricks ## Using Multiple Open AI Blocks: Tips and Tricks
In this video, I discuss some important things to keep in mind when using multiple Open AI blocks consecutively. I provide an example where we ask the user for a topic, generate a summary, and display a list of authors. In this video, I discuss some important things to keep in mind when using multiple Open AI blocks consecutively. I provide an example where we ask the user for a topic, generate a summary, and display a list of authors.

628
apps/docs/openapi/builder/_spec_.json
View File

@ -2996,6 +2996,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -6246,6 +6290,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -10332,6 +10420,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -14041,6 +14173,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -17901,6 +18077,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -21544,6 +21764,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -25275,6 +25539,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -29075,6 +29383,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -32797,6 +33149,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -36656,6 +37052,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -40266,6 +40706,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -44137,6 +44621,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -47816,6 +48344,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -51493,6 +52065,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -56665,6 +57281,18 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "type",
"in": "query",
"required": true,
"schema": {
"type": "string",
"enum": [
"gpt",
"tts"
]
}
} }
], ],
"responses": { "responses": {

264
apps/docs/openapi/chat/_spec_.json
View File

@ -2549,6 +2549,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -6144,6 +6188,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -13132,6 +13220,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -16727,6 +16859,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -29510,6 +29686,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }
@ -33105,6 +33325,50 @@
"responseMapping" "responseMapping"
], ],
"additionalProperties": false "additionalProperties": false
},
{
"type": "object",
"properties": {
"credentialsId": {
"type": "string"
},
"baseUrl": {
"type": "string"
},
"apiVersion": {
"type": "string"
},
"task": {
"type": "string",
"enum": [
"Create speech"
]
},
"model": {
"type": "string"
},
"input": {
"type": "string"
},
"voice": {
"type": "string",
"enum": [
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer"
]
},
"saveUrlInVariableId": {
"type": "string"
}
},
"required": [
"task"
],
"additionalProperties": false
} }
] ]
} }

4
apps/viewer/package.json
View File

@ -18,7 +18,7 @@
"@typebot.io/nextjs": "workspace:*", "@typebot.io/nextjs": "workspace:*",
"@typebot.io/js": "workspace:*", "@typebot.io/js": "workspace:*",
"@typebot.io/prisma": "workspace:*", "@typebot.io/prisma": "workspace:*",
"ai": "2.2.14", "ai": "2.2.24",
"bot-engine": "workspace:*", "bot-engine": "workspace:*",
"cors": "2.8.5", "cors": "2.8.5",
"google-spreadsheet": "4.0.2", "google-spreadsheet": "4.0.2",
@ -26,7 +26,7 @@
"next": "13.5.4", "next": "13.5.4",
"nextjs-cors": "2.1.2", "nextjs-cors": "2.1.2",
"nodemailer": "6.9.3", "nodemailer": "6.9.3",
"openai": "4.11.1", "openai": "4.19.0",
"qs": "6.11.2", "qs": "6.11.2",
"react": "18.2.0", "react": "18.2.0",
"react-dom": "18.2.0", "react-dom": "18.2.0",

121
packages/bot-engine/blocks/integrations/openai/audio/createSpeechOpenAI.ts Normal file
View File

@ -0,0 +1,121 @@
import { SessionState } from '@typebot.io/schemas'
import {
CreateSpeechOpenAIOptions,
OpenAICredentials,
} from '@typebot.io/schemas/features/blocks/integrations/openai'
import { isNotEmpty } from '@typebot.io/lib'
import { decrypt } from '@typebot.io/lib/api/encryption/decrypt'
import prisma from '@typebot.io/lib/prisma'
import { defaultOpenAIOptions } from '@typebot.io/schemas/features/blocks/integrations/openai/constants'
import { ExecuteIntegrationResponse } from '../../../../types'
import OpenAI, { ClientOptions } from 'openai'
import { uploadFileToBucket } from '@typebot.io/lib/s3/uploadFileToBucket'
import { updateVariablesInSession } from '../../../../variables/updateVariablesInSession'
import { createId } from '@paralleldrive/cuid2'
import { parseVariables } from '../../../../variables/parseVariables'
export const createSpeechOpenAI = async (
state: SessionState,
{
outgoingEdgeId,
options,
}: {
outgoingEdgeId?: string
options: CreateSpeechOpenAIOptions
}
): Promise<ExecuteIntegrationResponse> => {
let newSessionState = state
const noCredentialsError = {
status: 'error',
description: 'Make sure to select an OpenAI account',
}
if (!options.input || !options.voice || !options.saveUrlInVariableId) {
return {
outgoingEdgeId,
logs: [
{
status: 'error',
description:
'Make sure to enter an input, select a voice and select a variable to save the URL in',
},
],
}
}
if (!options.credentialsId) {
return {
outgoingEdgeId,
logs: [noCredentialsError],
}
}
const credentials = await prisma.credentials.findUnique({
where: {
id: options.credentialsId,
},
})
if (!credentials) {
console.error('Could not find credentials in database')
return { outgoingEdgeId, logs: [noCredentialsError] }
}
const { apiKey } = (await decrypt(
credentials.data,
credentials.iv
)) as OpenAICredentials['data']
const config = {
apiKey,
baseURL: options.baseUrl ?? defaultOpenAIOptions.baseUrl,
defaultHeaders: {
'api-key': apiKey,
},
defaultQuery: isNotEmpty(options.apiVersion)
? {
'api-version': options.apiVersion,
}
: undefined,
} satisfies ClientOptions
const openai = new OpenAI(config)
const variables = newSessionState.typebotsQueue[0].typebot.variables
const saveUrlInVariable = variables.find(
(v) => v.id === options.saveUrlInVariableId
)
if (!saveUrlInVariable) {
return {
outgoingEdgeId,
logs: [
{
status: 'error',
description: 'Could not find variable to save URL in',
},
],
}
}
const rawAudio = (await openai.audio.speech.create({
input: parseVariables(variables)(options.input),
voice: options.voice,
model: options.model as 'tts-1' | 'tts-1-hd',
})) as any
const url = await uploadFileToBucket({
file: Buffer.from((await rawAudio.arrayBuffer()) as ArrayBuffer),
key: `tmp/openai/audio/${createId() + createId()}.mp3`,
mimeType: 'audio/mpeg',
})
newSessionState = updateVariablesInSession(newSessionState)([
{
...saveUrlInVariable,
value: url,
},
])
return {
outgoingEdgeId,
newSessionState,
}
}

6
packages/bot-engine/blocks/integrations/openai/executeOpenAIBlock.ts
View File

@ -2,6 +2,7 @@ import { SessionState } from '@typebot.io/schemas'
import { OpenAIBlock } from '@typebot.io/schemas/features/blocks/integrations/openai' import { OpenAIBlock } from '@typebot.io/schemas/features/blocks/integrations/openai'
import { createChatCompletionOpenAI } from './createChatCompletionOpenAI' import { createChatCompletionOpenAI } from './createChatCompletionOpenAI'
import { ExecuteIntegrationResponse } from '../../../types' import { ExecuteIntegrationResponse } from '../../../types'
import { createSpeechOpenAI } from './audio/createSpeechOpenAI'
export const executeOpenAIBlock = async ( export const executeOpenAIBlock = async (
state: SessionState, state: SessionState,
@ -14,6 +15,11 @@ export const executeOpenAIBlock = async (
outgoingEdgeId: block.outgoingEdgeId, outgoingEdgeId: block.outgoingEdgeId,
blockId: block.id, blockId: block.id,
}) })
case 'Create speech':
return createSpeechOpenAI(state, {
options: block.options,
outgoingEdgeId: block.outgoingEdgeId,
})
case 'Create image': case 'Create image':
case undefined: case undefined:
return { outgoingEdgeId: block.outgoingEdgeId } return { outgoingEdgeId: block.outgoingEdgeId }

5
packages/bot-engine/blocks/integrations/openai/parseChatCompletionMessages.ts
View File

@ -103,11 +103,10 @@ export const parseChatCompletionMessages =
} satisfies OpenAI.Chat.ChatCompletionMessageParam } satisfies OpenAI.Chat.ChatCompletionMessageParam
}) })
.filter( .filter(
(message) => isNotEmpty(message?.role) && isNotEmpty(message?.content) (message) =>
isNotEmpty(message?.role) && isNotEmpty(message?.content?.toString())
) as OpenAI.Chat.ChatCompletionMessageParam[] ) as OpenAI.Chat.ChatCompletionMessageParam[]
console.log('parsedMessages', parsedMessages)
return { return {
variablesTransformedToList, variablesTransformedToList,
messages: parsedMessages, messages: parsedMessages,

4
packages/bot-engine/package.json
View File

@ -18,7 +18,7 @@
"@typebot.io/tsconfig": "workspace:*", "@typebot.io/tsconfig": "workspace:*",
"@udecode/plate-common": "21.1.5", "@udecode/plate-common": "21.1.5",
"@udecode/plate-serializer-md": "24.4.0", "@udecode/plate-serializer-md": "24.4.0",
"ai": "2.2.14", "ai": "2.2.24",
"chrono-node": "2.7.0", "chrono-node": "2.7.0",
"date-fns": "2.30.0", "date-fns": "2.30.0",
"google-auth-library": "8.9.0", "google-auth-library": "8.9.0",
@ -27,7 +27,7 @@
"libphonenumber-js": "1.10.37", "libphonenumber-js": "1.10.37",
"node-html-parser": "6.1.5", "node-html-parser": "6.1.5",
"nodemailer": "6.9.3", "nodemailer": "6.9.3",
"openai": "4.11.1", "openai": "4.19.0",
"qs": "6.11.2", "qs": "6.11.2",
"remark-slate": "1.8.6", "remark-slate": "1.8.6",
"stripe": "12.13.0" "stripe": "12.13.0"

15
packages/lib/s3/uploadFileToBucket.ts
View File

@ -2,13 +2,13 @@ import { env } from '@typebot.io/env'
import { Client } from 'minio' import { Client } from 'minio'
type Props = { type Props = {
fileName: string key: string
file: Buffer file: Buffer
mimeType: string mimeType: string
} }
export const uploadFileToBucket = async ({ export const uploadFileToBucket = async ({
fileName, key,
file, file,
mimeType, mimeType,
}: Props): Promise<string> => { }: Props): Promise<string> => {
@ -26,11 +26,14 @@ export const uploadFileToBucket = async ({
region: env.S3_REGION, region: env.S3_REGION,
}) })
await minioClient.putObject(env.S3_BUCKET, fileName, file, { await minioClient.putObject(env.S3_BUCKET, 'public/' + key, file, {
'Content-Type': mimeType, 'Content-Type': mimeType,
'Cache-Control': 'public, max-age=86400',
}) })
return `http${env.S3_SSL ? 's' : ''}://${env.S3_ENDPOINT}${ return env.S3_PUBLIC_CUSTOM_DOMAIN
env.S3_PORT ? `:${env.S3_PORT}` : '' ? `${env.S3_PUBLIC_CUSTOM_DOMAIN}/public/${key}`
}/${env.S3_BUCKET}/${fileName}` : `http${env.S3_SSL ? 's' : ''}://${env.S3_ENDPOINT}${
env.S3_PORT ? `:${env.S3_PORT}` : ''
}/${env.S3_BUCKET}/public/${key}`
} }

15
packages/schemas/features/blocks/integrations/openai/constants.ts
View File

@ -1,4 +1,8 @@
export const openAITasks = ['Create chat completion', 'Create image'] as const export const openAITasks = [
'Create chat completion',
'Create speech',
'Create image',
] as const
export const chatCompletionMessageRoles = [ export const chatCompletionMessageRoles = [
'system', 'system',
@ -27,3 +31,12 @@ export const defaultOpenAIOptions = {
export const defaultOpenAIResponseMappingItem = { export const defaultOpenAIResponseMappingItem = {
valueToExtract: 'Message content', valueToExtract: 'Message content',
} as const } as const
export const openAIVoices = [
'alloy',
'echo',
'fable',
'onyx',
'nova',
'shimmer',
] as const

26
packages/schemas/features/blocks/integrations/openai/schema.ts
View File

@ -4,6 +4,7 @@ import {
chatCompletionMessageRoles, chatCompletionMessageRoles,
chatCompletionResponseValues, chatCompletionResponseValues,
openAITasks, openAITasks,
openAIVoices,
} from './constants' } from './constants'
import { variableStringSchema } from '../../../utils' import { variableStringSchema } from '../../../utils'
import { blockBaseSchema, credentialsBaseSchema } from '../../shared' import { blockBaseSchema, credentialsBaseSchema } from '../../shared'
@ -78,10 +79,13 @@ const chatCompletionOptionsSchema = z
.optional(), .optional(),
}) })
.merge(openAIBaseOptionsSchema) .merge(openAIBaseOptionsSchema)
export type ChatCompletionOpenAIOptions = z.infer<
typeof chatCompletionOptionsSchema
>
const createImageOptionsSchema = z const createImageOptionsSchema = z
.object({ .object({
task: z.literal(openAITasks[1]), task: z.literal(openAITasks[2]),
prompt: z.string().optional(), prompt: z.string().optional(),
advancedOptions: z.object({ advancedOptions: z.object({
size: z.enum(['256x256', '512x512', '1024x1024']).optional(), size: z.enum(['256x256', '512x512', '1024x1024']).optional(),
@ -95,6 +99,18 @@ const createImageOptionsSchema = z
), ),
}) })
.merge(openAIBaseOptionsSchema) .merge(openAIBaseOptionsSchema)
export type CreateImageOpenAIOptions = z.infer<typeof createImageOptionsSchema>
const createSpeechOptionsSchema = openAIBaseOptionsSchema.extend({
task: z.literal(openAITasks[1]),
model: z.string().optional(),
input: z.string().optional(),
voice: z.enum(openAIVoices).optional(),
saveUrlInVariableId: z.string().optional(),
})
export type CreateSpeechOpenAIOptions = z.infer<
typeof createSpeechOptionsSchema
>
export const openAIBlockSchema = blockBaseSchema.merge( export const openAIBlockSchema = blockBaseSchema.merge(
z.object({ z.object({
@ -104,10 +120,12 @@ export const openAIBlockSchema = blockBaseSchema.merge(
initialOptionsSchema, initialOptionsSchema,
chatCompletionOptionsSchema, chatCompletionOptionsSchema,
createImageOptionsSchema, createImageOptionsSchema,
createSpeechOptionsSchema,
]) ])
.optional(), .optional(),
}) })
) )
export type OpenAIBlock = z.infer<typeof openAIBlockSchema>
export const openAICredentialsSchema = z export const openAICredentialsSchema = z
.object({ .object({
@ -117,10 +135,4 @@ export const openAICredentialsSchema = z
}), }),
}) })
.merge(credentialsBaseSchema) .merge(credentialsBaseSchema)
export type OpenAICredentials = z.infer<typeof openAICredentialsSchema> export type OpenAICredentials = z.infer<typeof openAICredentialsSchema>
export type OpenAIBlock = z.infer<typeof openAIBlockSchema>
export type ChatCompletionOpenAIOptions = z.infer<
typeof chatCompletionOptionsSchema
>
export type CreateImageOpenAIOptions = z.infer<typeof createImageOptionsSchema>

34
pnpm-lock.yaml generated
View File

@ -219,8 +219,8 @@ importers:
specifier: 0.2.0 specifier: 0.2.0
version: 0.2.0 version: 0.2.0
openai: openai:
specifier: 4.11.1 specifier: 4.19.0
version: 4.11.1 version: 4.19.0
papaparse: papaparse:
specifier: 5.4.1 specifier: 5.4.1
version: 5.4.1 version: 5.4.1
@ -534,8 +534,8 @@ importers:
specifier: workspace:* specifier: workspace:*
version: link:../../packages/prisma version: link:../../packages/prisma
ai: ai:
specifier: 2.2.14 specifier: 2.2.24
version: 2.2.14(react@18.2.0)(solid-js@1.7.8)(svelte@4.2.1)(vue@3.3.4) version: 2.2.24(react@18.2.0)(solid-js@1.7.8)(svelte@4.2.1)(vue@3.3.4)
bot-engine: bot-engine:
specifier: workspace:* specifier: workspace:*
version: link:../../packages/deprecated/bot-engine version: link:../../packages/deprecated/bot-engine
@ -558,8 +558,8 @@ importers:
specifier: 6.9.3 specifier: 6.9.3
version: 6.9.3 version: 6.9.3
openai: openai:
specifier: 4.11.1 specifier: 4.19.0
version: 4.11.1 version: 4.19.0
qs: qs:
specifier: 6.11.2 specifier: 6.11.2
version: 6.11.2 version: 6.11.2
@ -685,8 +685,8 @@ importers:
specifier: 24.4.0 specifier: 24.4.0
version: 24.4.0(react-dom@18.2.0)(react@18.2.0)(scheduler@0.23.0)(slate-history@0.93.0)(slate-hyperscript@0.77.0)(slate-react@0.99.0)(slate@0.94.1) version: 24.4.0(react-dom@18.2.0)(react@18.2.0)(scheduler@0.23.0)(slate-history@0.93.0)(slate-hyperscript@0.77.0)(slate-react@0.99.0)(slate@0.94.1)
ai: ai:
specifier: 2.2.14 specifier: 2.2.24
version: 2.2.14(react@18.2.0)(solid-js@1.7.8)(svelte@4.2.1)(vue@3.3.4) version: 2.2.24(react@18.2.0)(solid-js@1.7.8)(svelte@4.2.1)(vue@3.3.4)
chrono-node: chrono-node:
specifier: 2.7.0 specifier: 2.7.0
version: 2.7.0 version: 2.7.0
@ -712,8 +712,8 @@ importers:
specifier: 6.9.3 specifier: 6.9.3
version: 6.9.3 version: 6.9.3
openai: openai:
specifier: 4.11.1 specifier: 4.19.0
version: 4.11.1 version: 4.19.0
qs: qs:
specifier: 6.11.2 specifier: 6.11.2
version: 6.11.2 version: 6.11.2
@ -11247,8 +11247,8 @@ packages:
indent-string: 4.0.0 indent-string: 4.0.0
dev: false dev: false
/ai@2.2.14(react@18.2.0)(solid-js@1.7.8)(svelte@4.2.1)(vue@3.3.4): /ai@2.2.24(react@18.2.0)(solid-js@1.7.8)(svelte@4.2.1)(vue@3.3.4):
resolution: {integrity: sha512-4kL2iYPVhH1pl6jJFIJCYcgx5mHzGOmdwiSYWVadmSkNOxKqokgevHyJKiyL9B9DjlreM9cDqkQop56Hdfkb0w==} resolution: {integrity: sha512-j9f89SC08SxN50wKp4ccIGniUMtCsslvOBbYGBCSIvDbMBQaNTZuKW4Id8tmfEeNMV9mwZZ5wRNj6/1Z/pFwYA==}
engines: {node: '>=14.6'} engines: {node: '>=14.6'}
peerDependencies: peerDependencies:
react: ^18.2.0 react: ^18.2.0
@ -18905,8 +18905,8 @@ packages:
is-wsl: 2.2.0 is-wsl: 2.2.0
dev: false dev: false
/openai@4.11.1: /openai@4.19.0:
resolution: {integrity: sha512-GU0HQWbejXuVAQlDjxIE8pohqnjptFDIm32aPlNT1H9ucMz1VJJD0DaTJRQsagNaJ97awWjjVLEG7zCM6sm4SA==} resolution: {integrity: sha512-cJbl0noZyAaXVKBTMMq6X5BAvP1pm2rWYDBnZes99NL+Zh5/4NmlAwyuhTZEru5SqGGZIoiYKeMPXy4bm9DI0w==}
hasBin: true hasBin: true
dependencies: dependencies:
'@types/node': 18.11.18 '@types/node': 18.11.18
@ -18917,6 +18917,7 @@ packages:
form-data-encoder: 1.7.2 form-data-encoder: 1.7.2
formdata-node: 4.4.1 formdata-node: 4.4.1
node-fetch: 2.7.0 node-fetch: 2.7.0
web-streams-polyfill: 3.2.1
transitivePeerDependencies: transitivePeerDependencies:
- encoding - encoding
dev: false dev: false
@ -23678,6 +23679,11 @@ packages:
transitivePeerDependencies: transitivePeerDependencies:
- encoding - encoding
/web-streams-polyfill@3.2.1:
resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
engines: {node: '>= 8'}
dev: false
/web-streams-polyfill@4.0.0-beta.3: /web-streams-polyfill@4.0.0-beta.3:
resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==}
engines: {node: '>= 14'} engines: {node: '>= 14'}