diff --git a/apps/builder/src/components/inputs/CodeEditor.tsx b/apps/builder/src/components/inputs/CodeEditor.tsx index a569d0703..267d523de 100644 --- a/apps/builder/src/components/inputs/CodeEditor.tsx +++ b/apps/builder/src/components/inputs/CodeEditor.tsx @@ -1,11 +1,15 @@ import { BoxProps, Fade, + FormControl, + FormHelperText, + FormLabel, HStack, + Stack, useColorModeValue, useDisclosure, } from '@chakra-ui/react' -import { useEffect, useRef, useState } from 'react' +import React, { ReactNode, useEffect, useRef, useState } from 'react' import { useDebouncedCallback } from 'use-debounce' import { VariablesButton } from '@/features/variables/components/VariablesButton' import { Variable } from '@typebot.io/schemas' @@ -16,8 +20,10 @@ import { githubLight } from '@uiw/codemirror-theme-github' import { LanguageName, loadLanguage } from '@uiw/codemirror-extensions-langs' import { isDefined } from '@udecode/plate-common' import { CopyButton } from '../CopyButton' +import { MoreInfoTooltip } from '../MoreInfoTooltip' type Props = { + label?: string value?: string defaultValue?: string lang: LanguageName @@ -27,11 +33,18 @@ type Props = { height?: string maxHeight?: string minWidth?: string + moreInfoTooltip?: string + helperText?: ReactNode + isRequired?: boolean onChange?: (value: string) => void } export const CodeEditor = ({ + label, defaultValue, lang, + moreInfoTooltip, + helperText, + isRequired, onChange, height = '250px', maxHeight = '70vh', @@ -86,71 +99,91 @@ export const CodeEditor = ({ ) return ( - - - {isVariableButtonDisplayed && ( - + {label && ( + + {label}{' '} + {moreInfoTooltip && ( + {moreInfoTooltip} + )} + )} - {isReadOnly && ( - - + + {isVariableButtonDisplayed && ( + - - )} - + )} + {isReadOnly && ( + + + + )} + + {helperText && {helperText}} + ) } diff --git a/apps/builder/src/features/forge/components/zodLayouts/ZodFieldLayout.tsx b/apps/builder/src/features/forge/components/zodLayouts/ZodFieldLayout.tsx index bf9cebb6c..53bf53a5a 100644 --- a/apps/builder/src/features/forge/components/zodLayouts/ZodFieldLayout.tsx +++ b/apps/builder/src/features/forge/components/zodLayouts/ZodFieldLayout.tsx @@ -22,6 +22,7 @@ import { DropdownList } from '@/components/DropdownList' import { ForgedBlockDefinition, ForgedBlock } from '@typebot.io/forge-schemas' import { PrimitiveList } from '@/components/PrimitiveList' import { SwitchWithLabel } from '@/components/inputs/SwitchWithLabel' +import { CodeEditor } from '@/components/inputs/CodeEditor' const mdComponents = { a: ({ href, children }) => ( @@ -99,6 +100,8 @@ export const ZodFieldLayout = ({ @@ -229,6 +234,28 @@ export const ZodFieldLayout = ({ /> ) } + + if (layout?.inputType === 'code') + return ( + + {layout.helperText} + + ) : undefined + } + isRequired={layout?.isRequired} + withVariableButton={layout?.withVariableButton} + moreInfoTooltip={layout.moreInfoTooltip} + onChange={onDataChange} + width={width} + /> + ) return ( | undefined isInAccordion?: boolean onDataChange: (val: any) => void @@ -283,6 +314,8 @@ const ZodArrayContent = ({ {({ item, onItemChange }) => ( - + +### Tools + +The tools section allows you to add functions that the OpenAI model can execute. Here is an example of a function named `getWeather` that returns 'Sunny and warm' if you ask about the weather of Paris and 'Rainy and cold' if you ask for any other city 😂. + +A more useful example would be, of course, to call an API to get the weather of the city the user is asking about. + + + OpenAI tools + + +As you can see, the code block expects the body of the Javascript function. You can use the `return` keyword to return values. + ## Ask assistant This action allows you to talk with your [OpenAI assistant](https://platform.openai.com/assistants). All you have to do is to provide its ID. @@ -44,6 +56,10 @@ This action allows you to talk with your [OpenAI assistant](https://platform.ope /> +### Functions + +If you defined functions in your assistant, you can define the function to execute in the `Functions` section. + ## Create speech This action allows you to transform a text input into an audio URL that you can reuse in your bot. diff --git a/apps/docs/images/blocks/integrations/openai/tools.png b/apps/docs/images/blocks/integrations/openai/tools.png new file mode 100644 index 000000000..c06e18ac4 Binary files /dev/null and b/apps/docs/images/blocks/integrations/openai/tools.png differ diff --git a/packages/embeds/js/package.json b/packages/embeds/js/package.json index 3f55b2f14..2f06129b3 100644 --- a/packages/embeds/js/package.json +++ b/packages/embeds/js/package.json @@ -1,6 +1,6 @@ { "name": "@typebot.io/js", - "version": "0.2.33", + "version": "0.2.34", "description": "Javascript library to display typebots on your website", "type": "module", "main": "dist/index.js", diff --git a/packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx b/packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx index ca8b7f96c..2367d0104 100644 --- a/packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx +++ b/packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx @@ -90,27 +90,7 @@ export const ConversationContainer = (props: Props) => { const actionsBeforeFirstBubble = initialChunk.clientSideActions.filter( (action) => isNotDefined(action.lastBubbleBlockId) ) - for (const action of actionsBeforeFirstBubble) { - if ( - 'streamOpenAiChatCompletion' in action || - 'webhookToExecute' in action - ) - setIsSending(true) - const response = await executeClientSideAction({ - clientSideAction: action, - context: { - apiHost: props.context.apiHost, - sessionId: props.initialChatReply.sessionId, - }, - onMessageStream: streamMessage, - }) - if (response && 'replyToSend' in response) { - sendMessage(response.replyToSend, response.logs) - return - } - if (response && 'blockedPopupUrl' in response) - setBlockedPopupUrl(response.blockedPopupUrl) - } + processClientSideActions(actionsBeforeFirstBubble) })() }) @@ -210,27 +190,7 @@ export const ConversationContainer = (props: Props) => { const actionsBeforeFirstBubble = data.clientSideActions.filter((action) => isNotDefined(action.lastBubbleBlockId) ) - for (const action of actionsBeforeFirstBubble) { - if ( - 'streamOpenAiChatCompletion' in action || - 'webhookToExecute' in action - ) - setIsSending(true) - const response = await executeClientSideAction({ - clientSideAction: action, - context: { - apiHost: props.context.apiHost, - sessionId: props.initialChatReply.sessionId, - }, - onMessageStream: streamMessage, - }) - if (response && 'replyToSend' in response) { - sendMessage(response.replyToSend, response.logs) - return - } - if (response && 'blockedPopupUrl' in response) - setBlockedPopupUrl(response.blockedPopupUrl) - } + processClientSideActions(actionsBeforeFirstBubble) } setChatChunks((displayedChunks) => [ ...displayedChunks, @@ -267,27 +227,35 @@ export const ConversationContainer = (props: Props) => { const actionsToExecute = lastChunk.clientSideActions.filter( (action) => action.lastBubbleBlockId === blockId ) - for (const action of actionsToExecute) { - if ( - 'streamOpenAiChatCompletion' in action || - 'webhookToExecute' in action - ) - setIsSending(true) - const response = await executeClientSideAction({ - clientSideAction: action, - context: { - apiHost: props.context.apiHost, - sessionId: props.initialChatReply.sessionId, - }, - onMessageStream: streamMessage, - }) - if (response && 'replyToSend' in response) { - sendMessage(response.replyToSend, response.logs) - return - } - if (response && 'blockedPopupUrl' in response) - setBlockedPopupUrl(response.blockedPopupUrl) + await processClientSideActions(actionsToExecute) + } + } + + const processClientSideActions = async ( + actions: NonNullable + ) => { + for (const action of actions) { + if ( + 'streamOpenAiChatCompletion' in action || + 'webhookToExecute' in action || + 'stream' in action + ) + setIsSending(true) + const response = await executeClientSideAction({ + clientSideAction: action, + context: { + apiHost: props.context.apiHost, + sessionId: props.initialChatReply.sessionId, + }, + onMessageStream: streamMessage, + }) + if (response && 'replyToSend' in response) { + setIsSending(false) + sendMessage(response.replyToSend, response.logs) + return } + if (response && 'blockedPopupUrl' in response) + setBlockedPopupUrl(response.blockedPopupUrl) } } diff --git a/packages/embeds/nextjs/package.json b/packages/embeds/nextjs/package.json index 3393d9e0d..59b63c583 100644 --- a/packages/embeds/nextjs/package.json +++ b/packages/embeds/nextjs/package.json @@ -1,6 +1,6 @@ { "name": "@typebot.io/nextjs", - "version": "0.2.33", + "version": "0.2.34", "description": "Convenient library to display typebots on your Next.js website", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/embeds/react/package.json b/packages/embeds/react/package.json index 14be32669..3b934cf15 100644 --- a/packages/embeds/react/package.json +++ b/packages/embeds/react/package.json @@ -1,6 +1,6 @@ { "name": "@typebot.io/react", - "version": "0.2.33", + "version": "0.2.34", "description": "Convenient library to display typebots on your React app", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/forge/blocks/openai/actions/askAssistant.tsx b/packages/forge/blocks/openai/actions/askAssistant.tsx index e48dbbeb9..25f63302b 100644 --- a/packages/forge/blocks/openai/actions/askAssistant.tsx +++ b/packages/forge/blocks/openai/actions/askAssistant.tsx @@ -24,6 +24,24 @@ export const askAssistant = createAction({ label: 'Message', inputType: 'textarea', }), + functions: option + .array( + option.object({ + name: option.string.layout({ + fetcher: 'fetchAssistantFunctions', + label: 'Name', + }), + code: option.string.layout({ + inputType: 'code', + label: 'Code', + lang: 'javascript', + moreInfoTooltip: + 'A javascript code snippet that can use the defined parameters. It should return a value.', + withVariableButton: false, + }), + }) + ) + .layout({ accordion: 'Functions', itemLabel: 'function' }), responseMapping: option .saveResponseArray(['Message', 'Thread ID'] as const) .layout({ @@ -64,6 +82,40 @@ export const askAssistant = createAction({ }, dependencies: ['baseUrl', 'apiVersion'], }, + { + id: 'fetchAssistantFunctions', + fetch: async ({ options, credentials }) => { + if (!options.assistantId) return [] + const config = { + apiKey: credentials.apiKey, + baseURL: options.baseUrl, + defaultHeaders: { + 'api-key': credentials.apiKey, + }, + defaultQuery: options.apiVersion + ? { + 'api-version': options.apiVersion, + } + : undefined, + } satisfies ClientOptions + + const openai = new OpenAI(config) + + const response = await openai.beta.assistants.retrieve( + options.assistantId + ) + + return response.tools + .filter((tool) => tool.type === 'function') + .map((tool) => + tool.type === 'function' && tool.function.name + ? tool.function.name + : undefined + ) + .filter(isDefined) + }, + dependencies: ['baseUrl', 'apiVersion', 'assistantId'], + }, ], getSetVariableIds: ({ responseMapping }) => responseMapping?.map((r) => r.variableId).filter(isDefined) ?? [], @@ -77,6 +129,7 @@ export const askAssistant = createAction({ message, responseMapping, threadId, + functions, }, variables, logs, @@ -139,6 +192,45 @@ export const askAssistant = createAction({ ) { throw new Error(run.status) } + if (run.status === 'requires_action') { + if (run.required_action?.type === 'submit_tool_outputs') { + const tool_outputs = ( + await Promise.all( + run.required_action.submit_tool_outputs.tool_calls.map( + async (toolCall) => { + const parameters = JSON.parse(toolCall.function.arguments) + + const functionToExecute = functions?.find( + (f) => f.name === toolCall.function.name + ) + if (!functionToExecute) return + + const name = toolCall.function.name + if (!name) return + const func = AsyncFunction( + ...Object.keys(parameters), + functionToExecute.code + ) + const output = await func(...Object.values(parameters)) + + return { + tool_call_id: toolCall.id, + output, + } + } + ) + ) + ).filter(isDefined) + + run = await openai.beta.threads.runs.submitToolOutputs( + currentThreadId, + run.id, + { tool_outputs } + ) + + await waitForRun(run) + } + } } await waitForRun(run) @@ -170,3 +262,5 @@ export const askAssistant = createAction({ }, }, }) + +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor diff --git a/packages/forge/blocks/openai/actions/createChatCompletion.tsx b/packages/forge/blocks/openai/actions/createChatCompletion.tsx index b4d203d71..8df72fcb5 100644 --- a/packages/forge/blocks/openai/actions/createChatCompletion.tsx +++ b/packages/forge/blocks/openai/actions/createChatCompletion.tsx @@ -1,11 +1,13 @@ import { option, createAction } from '@typebot.io/forge' import OpenAI, { ClientOptions } from 'openai' import { defaultOpenAIOptions } from '../constants' -import { OpenAIStream } from 'ai' +import { OpenAIStream, ToolCallPayload } from 'ai' import { parseChatCompletionMessages } from '../helpers/parseChatCompletionMessages' import { isDefined } from '@typebot.io/lib' import { auth } from '../auth' import { baseOptions } from '../baseOptions' +import { ChatCompletionTool } from 'openai/resources/chat/completions' +import { parseToolParameters } from '../helpers/parseToolParameters' const nativeMessageContentSchema = { content: option.string.layout({ @@ -32,6 +34,48 @@ const assistantMessageItemSchema = option }) .extend(nativeMessageContentSchema) +export const parameterSchema = option.object({ + name: option.string.layout({ + label: 'Name', + placeholder: 'myVariable', + withVariableButton: false, + }), + type: option.enum(['string', 'number', 'boolean']).layout({ + label: 'Type:', + direction: 'row', + }), + description: option.string.layout({ + label: 'Description', + withVariableButton: false, + }), + required: option.boolean.layout({ + label: 'Is required?', + }), +}) + +const functionToolItemSchema = option.object({ + type: option.literal('function'), + name: option.string.layout({ + label: 'Name', + placeholder: 'myFunctionName', + withVariableButton: false, + }), + description: option.string.layout({ + label: 'Description', + placeholder: 'A brief description of what this function does.', + withVariableButton: false, + }), + parameters: option.array(parameterSchema).layout({ accordion: 'Parameters' }), + code: option.string.layout({ + inputType: 'code', + label: 'Code', + lang: 'javascript', + moreInfoTooltip: + 'A javascript code snippet that can use the defined parameters. It should return a value.', + withVariableButton: false, + }), +}) + const dialogueMessageItemSchema = option.object({ role: option.literal('Dialogue'), dialogueVariableId: option.string.layout({ @@ -61,6 +105,9 @@ export const options = option.object({ ]) ) .layout({ accordion: 'Messages', itemLabel: 'message', isOrdered: true }), + tools: option + .array(option.discriminatedUnion('type', [functionToolItemSchema])) + .layout({ accordion: 'Tools', itemLabel: 'tool' }), temperature: option.number.layout({ accordion: 'Advanced settings', label: 'Temperature', @@ -131,20 +178,74 @@ export const createChatCompletion = createAction({ const openai = new OpenAI(config) + const tools = options.tools + ? (options.tools + .filter((t) => t.name && t.parameters) + .map((t) => ({ + type: 'function', + function: { + name: t.name as string, + description: t.description, + parameters: parseToolParameters(t.parameters!), + }, + })) satisfies ChatCompletionTool[]) + : undefined + + const messages = parseChatCompletionMessages({ options, variables }) + const response = await openai.chat.completions.create({ model: options.model ?? defaultOpenAIOptions.model, temperature: options.temperature ? Number(options.temperature) : undefined, - messages: parseChatCompletionMessages({ options, variables }), + messages, + tools, }) + let message = response.choices[0].message + let totalTokens = response.usage?.total_tokens + + if (message.tool_calls) { + messages.push(message) + + for (const toolCall of message.tool_calls) { + const name = toolCall.function?.name + if (!name) continue + const toolDefinition = options.tools?.find((t) => t.name === name) + if (!toolDefinition?.code || !toolDefinition.parameters) continue + const func = AsyncFunction( + ...toolDefinition.parameters?.map((p) => p.name), + toolDefinition.code + ) + const content = await func( + ...Object.values(JSON.parse(toolCall.function.arguments)) + ) + messages.push({ + tool_call_id: toolCall.id, + role: 'tool', + content, + }) + } + + const secondResponse = await openai.chat.completions.create({ + model: options.model ?? defaultOpenAIOptions.model, + temperature: options.temperature + ? Number(options.temperature) + : undefined, + messages, + tools, + }) + + message = secondResponse.choices[0].message + totalTokens = secondResponse.usage?.total_tokens + } + options.responseMapping?.forEach((mapping) => { if (!mapping.variableId) return if (!mapping.item || mapping.item === 'Message content') - variables.set(mapping.variableId, response.choices[0].message.content) + variables.set(mapping.variableId, message.content) if (mapping.item === 'Total tokens') - variables.set(mapping.variableId, response.usage?.total_tokens) + variables.set(mapping.variableId, totalTokens) }) }, stream: { @@ -168,17 +269,70 @@ export const createChatCompletion = createAction({ const openai = new OpenAI(config) + const tools = options.tools + ? (options.tools + .filter((t) => t.name && t.parameters) + .map((t) => ({ + type: 'function', + function: { + name: t.name as string, + description: t.description, + parameters: parseToolParameters(t.parameters!), + }, + })) satisfies ChatCompletionTool[]) + : undefined + + const messages = parseChatCompletionMessages({ options, variables }) + const response = await openai.chat.completions.create({ model: options.model ?? defaultOpenAIOptions.model, temperature: options.temperature ? Number(options.temperature) : undefined, stream: true, - messages: parseChatCompletionMessages({ options, variables }), + messages, + tools, }) - return OpenAIStream(response) + return OpenAIStream(response, { + experimental_onToolCall: async ( + call: ToolCallPayload, + appendToolCallMessage + ) => { + for (const toolCall of call.tools) { + const name = toolCall.func?.name + if (!name) continue + const toolDefinition = options.tools?.find((t) => t.name === name) + if (!toolDefinition?.code || !toolDefinition.parameters) continue + const func = AsyncFunction( + ...toolDefinition.parameters?.map((p) => p.name), + toolDefinition.code + ) + const content = await func( + ...Object.values(JSON.parse(toolCall.func.arguments as any)) + ) + + const newMessages = appendToolCallMessage({ + tool_call_id: toolCall.id, + function_name: toolCall.func.name, + tool_call_result: content, + }) + + return openai.chat.completions.create({ + messages: [ + ...messages, + ...newMessages, + ] as OpenAI.Chat.Completions.ChatCompletionMessageParam[], + model: options.model ?? defaultOpenAIOptions.model, + stream: true, + tools, + }) + } + }, + }) }, }, }, }) + +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor diff --git a/packages/forge/blocks/openai/helpers/parseToolParameters.ts b/packages/forge/blocks/openai/helpers/parseToolParameters.ts new file mode 100644 index 000000000..a29e8949e --- /dev/null +++ b/packages/forge/blocks/openai/helpers/parseToolParameters.ts @@ -0,0 +1,22 @@ +import type { OpenAI } from 'openai' +import { parameterSchema } from '../actions/createChatCompletion' +import { z } from '@typebot.io/forge/zod' + +export const parseToolParameters = ( + parameters: z.infer[] +): OpenAI.FunctionParameters => ({ + type: 'object', + properties: parameters?.reduce<{ + [x: string]: unknown + }>((acc, param) => { + if (!param.name) return acc + acc[param.name] = { + type: param.type, + description: param.description, + } + return acc + }, {}), + required: + parameters?.filter((param) => param.required).map((param) => param.name) ?? + [], +}) diff --git a/packages/forge/core/zod/extendWithTypebotLayout.ts b/packages/forge/core/zod/extendWithTypebotLayout.ts index dcbabd351..36c481e5a 100644 --- a/packages/forge/core/zod/extendWithTypebotLayout.ts +++ b/packages/forge/core/zod/extendWithTypebotLayout.ts @@ -8,7 +8,8 @@ export interface ZodLayoutMetadata< > { accordion?: string label?: string - inputType?: 'variableDropdown' | 'textarea' | 'password' + inputType?: 'variableDropdown' | 'textarea' | 'password' | 'code' + lang?: 'javascript' | 'html' | 'css' | 'json' defaultValue?: T extends ZodDate ? string : TInferred placeholder?: string helperText?: string