✨ (openai) Add tools and functions support (#1167)

Closes #863

Got helped from #1162 for the implementation. Closing it in favor of
this PR.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Enhanced `CodeEditor` with additional properties for better form
control and validation.
- Introduced tools and functions in OpenAI integrations documentation
for custom JavaScript execution.
- Added capability to define and use custom JavaScript functions with
the OpenAI assistant.
- Expanded layout metadata options to include various input types and
languages.

- **Improvements**
- Updated the OpenAI actions to support new function execution features.

- **Documentation**
- Added new sections for tools and functions in the OpenAI integrations
guide.

- **Refactor**
- Refactored components and actions to integrate new features and
improve existing functionalities.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

apps/builder/src/components/inputs/CodeEditor.tsx

@@ -1,11 +1,15 @@
 import {
   BoxProps,
   Fade,
+  FormControl,
+  FormHelperText,
+  FormLabel,
   HStack,
+  Stack,
   useColorModeValue,
   useDisclosure,
 } from '@chakra-ui/react'
-import { useEffect, useRef, useState } from 'react'
+import React, { ReactNode, useEffect, useRef, useState } from 'react'
 import { useDebouncedCallback } from 'use-debounce'
 import { VariablesButton } from '@/features/variables/components/VariablesButton'
 import { Variable } from '@typebot.io/schemas'
@@ -16,8 +20,10 @@ import { githubLight } from '@uiw/codemirror-theme-github'
 import { LanguageName, loadLanguage } from '@uiw/codemirror-extensions-langs'
 import { isDefined } from '@udecode/plate-common'
 import { CopyButton } from '../CopyButton'
+import { MoreInfoTooltip } from '../MoreInfoTooltip'
 
 type Props = {
+  label?: string
   value?: string
   defaultValue?: string
   lang: LanguageName
@@ -27,11 +33,18 @@ type Props = {
   height?: string
   maxHeight?: string
   minWidth?: string
+  moreInfoTooltip?: string
+  helperText?: ReactNode
+  isRequired?: boolean
   onChange?: (value: string) => void
 }
 export const CodeEditor = ({
+  label,
   defaultValue,
   lang,
+  moreInfoTooltip,
+  helperText,
+  isRequired,
   onChange,
   height = '250px',
   maxHeight = '70vh',
@@ -86,71 +99,91 @@ export const CodeEditor = ({
   )
 
   return (
-    <HStack
+    <FormControl
-      align="flex-end"
+      isRequired={isRequired}
-      spacing={0}
+      as={Stack}
-      borderWidth={'1px'}
+      justifyContent="space-between"
-      rounded="md"
+      spacing={2}
-      bg={useColorModeValue('white', '#1A1B26')}
+      flex="1"
-      width="full"
-      h="full"
-      pos="relative"
-      minW={minWidth}
-      onMouseEnter={onOpen}
-      onMouseLeave={onClose}
-      maxWidth={props.maxWidth}
-      sx={{
-        '& .cm-editor': {
-          maxH: maxHeight,
-          outline: '0px solid transparent !important',
-          rounded: 'md',
-        },
-        '& .cm-scroller': {
-          rounded: 'md',
-          overflow: 'auto',
-        },
-        '& .cm-gutter,.cm-content': {
-          minH: isReadOnly ? '0' : height,
-        },
-        '& .ͼ1 .cm-scroller': {
-          fontSize: '14px',
-          fontFamily:
-            'JetBrainsMono, SFMono-Regular, SF Mono, Menlo, Consolas, Liberation Mono, monospace',
-        },
-      }}
     >
-      <CodeMirror
+      {label && (
-        data-testid="code-editor"
+        <FormLabel display="flex" flexShrink={0} gap="1" mb="0" mr="0">
-        ref={codeEditor}
+          {label}{' '}
-        value={props.value ?? value}
+          {moreInfoTooltip && (
-        onChange={handleChange}
+            <MoreInfoTooltip>{moreInfoTooltip}</MoreInfoTooltip>
-        onBlur={rememberCarretPosition}
+          )}
-        theme={theme}
+        </FormLabel>
-        extensions={[loadLanguage(lang)].filter(isDefined)}
-        editable={!isReadOnly}
-        style={{
-          width: isVariableButtonDisplayed ? 'calc(100% - 32px)' : '100%',
-        }}
-        spellCheck={false}
-        basicSetup={{
-          highlightActiveLine: false,
-        }}
-      />
-      {isVariableButtonDisplayed && (
-        <VariablesButton onSelectVariable={handleVariableSelected} size="sm" />
       )}
-      {isReadOnly && (
+      <HStack
-        <Fade in={isOpen}>
+        align="flex-end"
-          <CopyButton
+        spacing={0}
-            textToCopy={props.value ?? value}
+        borderWidth={'1px'}
-            pos="absolute"
+        rounded="md"
-            right={0.5}
+        bg={useColorModeValue('white', '#1A1B26')}
-            top={0.5}
+        width="full"
-            size="xs"
+        h="full"
-            colorScheme="blue"
+        pos="relative"
+        minW={minWidth}
+        onMouseEnter={onOpen}
+        onMouseLeave={onClose}
+        maxWidth={props.maxWidth}
+        sx={{
+          '& .cm-editor': {
+            maxH: maxHeight,
+            outline: '0px solid transparent !important',
+            rounded: 'md',
+          },
+          '& .cm-scroller': {
+            rounded: 'md',
+            overflow: 'auto',
+          },
+          '& .cm-gutter,.cm-content': {
+            minH: isReadOnly ? '0' : height,
+          },
+          '& .ͼ1 .cm-scroller': {
+            fontSize: '14px',
+            fontFamily:
+              'JetBrainsMono, SFMono-Regular, SF Mono, Menlo, Consolas, Liberation Mono, monospace',
+          },
+        }}
+      >
+        <CodeMirror
+          data-testid="code-editor"
+          ref={codeEditor}
+          value={props.value ?? value}
+          onChange={handleChange}
+          onBlur={rememberCarretPosition}
+          theme={theme}
+          extensions={[loadLanguage(lang)].filter(isDefined)}
+          editable={!isReadOnly}
+          style={{
+            width: isVariableButtonDisplayed ? 'calc(100% - 32px)' : '100%',
+          }}
+          spellCheck={false}
+          basicSetup={{
+            highlightActiveLine: false,
+          }}
+        />
+        {isVariableButtonDisplayed && (
+          <VariablesButton
+            onSelectVariable={handleVariableSelected}
+            size="sm"
           />
-        </Fade>
+        )}
-      )}
+        {isReadOnly && (
-    </HStack>
+          <Fade in={isOpen}>
+            <CopyButton
+              textToCopy={props.value ?? value}
+              pos="absolute"
+              right={0.5}
+              top={0.5}
+              size="xs"
+              colorScheme="blue"
+            />
+          </Fade>
+        )}
+      </HStack>
+      {helperText && <FormHelperText mt="0">{helperText}</FormHelperText>}
+    </FormControl>
   )
 }

apps/builder/src/features/forge/components/zodLayouts/ZodFieldLayout.tsx

@@ -22,6 +22,7 @@ import { DropdownList } from '@/components/DropdownList'
 import { ForgedBlockDefinition, ForgedBlock } from '@typebot.io/forge-schemas'
 import { PrimitiveList } from '@/components/PrimitiveList'
 import { SwitchWithLabel } from '@/components/inputs/SwitchWithLabel'
+import { CodeEditor } from '@/components/inputs/CodeEditor'
 
 const mdComponents = {
   a: ({ href, children }) => (
@@ -99,6 +100,8 @@ export const ZodFieldLayout = ({
                 <ZodArrayContent
                   data={data}
                   schema={schema}
+                  blockDef={blockDef}
+                  blockOptions={blockOptions}
                   layout={layout}
                   onDataChange={onDataChange}
                   isInAccordion
@@ -111,6 +114,8 @@ export const ZodFieldLayout = ({
         <ZodArrayContent
           data={data}
           schema={schema}
+          blockDef={blockDef}
+          blockOptions={blockOptions}
           layout={layout}
           onDataChange={onDataChange}
         />
@@ -229,6 +234,28 @@ export const ZodFieldLayout = ({
           />
         )
       }
+
+      if (layout?.inputType === 'code')
+        return (
+          <CodeEditor
+            defaultValue={data ?? layout?.defaultValue}
+            lang={layout.lang ?? 'javascript'}
+            label={layout?.label}
+            placeholder={layout?.placeholder}
+            helperText={
+              layout?.helperText ? (
+                <Markdown components={mdComponents}>
+                  {layout.helperText}
+                </Markdown>
+              ) : undefined
+            }
+            isRequired={layout?.isRequired}
+            withVariableButton={layout?.withVariableButton}
+            moreInfoTooltip={layout.moreInfoTooltip}
+            onChange={onDataChange}
+            width={width}
+          />
+        )
       return (
         <TextInput
           defaultValue={data ?? layout?.defaultValue}
@@ -254,12 +281,16 @@ export const ZodFieldLayout = ({
 const ZodArrayContent = ({
   schema,
   data,
+  blockDef,
+  blockOptions,
   layout,
   isInAccordion,
   onDataChange,
 }: {
   schema: z.ZodTypeAny
   data: any
+  blockDef?: ForgedBlockDefinition
+  blockOptions?: ForgedBlock['options']
   layout: ZodLayoutMetadata<ZodTypeAny> | undefined
   isInAccordion?: boolean
   onDataChange: (val: any) => void
@@ -283,6 +314,8 @@ const ZodArrayContent = ({
               <ZodFieldLayout
                 schema={schema._def.innerType._def.type}
                 data={item}
+                blockDef={blockDef}
+                blockOptions={blockOptions}
                 isInAccordion={isInAccordion}
                 onDataChange={onItemChange}
                 width="full"
@@ -302,9 +335,11 @@ const ZodArrayContent = ({
       isOrdered={layout?.isOrdered}
     >
       {({ item, onItemChange }) => (
-        <Stack p="4" rounded="md" flex="1" borderWidth="1px">
+        <Stack p="4" rounded="md" flex="1" borderWidth="1px" maxW="100%">
           <ZodFieldLayout
             schema={schema._def.innerType._def.type}
+            blockDef={blockDef}
+            blockOptions={blockOptions}
             data={item}
             isInAccordion={isInAccordion}
             onDataChange={onItemChange}

apps/docs/editor/blocks/integrations/openai.mdx

@@ -33,6 +33,18 @@ Then you can give the OpenAI block access to this sequence of messages:
   />
 </Frame>
 
+### Tools
+
+The tools section allows you to add functions that the OpenAI model can execute. Here is an example of a function named `getWeather` that returns 'Sunny and warm' if you ask about the weather of Paris and 'Rainy and cold' if you ask for any other city 😂.
+
+A more useful example would be, of course, to call an API to get the weather of the city the user is asking about.
+
+<Frame>
+  <img src="/images/blocks/integrations/openai/tools.png" alt="OpenAI tools" />
+</Frame>
+
+As you can see, the code block expects the body of the Javascript function. You can use the `return` keyword to return values.
+
 ## Ask assistant
 
 This action allows you to talk with your [OpenAI assistant](https://platform.openai.com/assistants). All you have to do is to provide its ID.
@@ -44,6 +56,10 @@ This action allows you to talk with your [OpenAI assistant](https://platform.ope
   />
 </Frame>
 
+### Functions
+
+If you defined functions in your assistant, you can define the function to execute in the `Functions` section.
+
 ## Create speech
 
 This action allows you to transform a text input into an audio URL that you can reuse in your bot.

packages/embeds/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/js",
-  "version": "0.2.33",
+  "version": "0.2.34",
   "description": "Javascript library to display typebots on your website",
   "type": "module",
   "main": "dist/index.js",

packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx

@@ -90,27 +90,7 @@ export const ConversationContainer = (props: Props) => {
       const actionsBeforeFirstBubble = initialChunk.clientSideActions.filter(
         (action) => isNotDefined(action.lastBubbleBlockId)
       )
-      for (const action of actionsBeforeFirstBubble) {
+      processClientSideActions(actionsBeforeFirstBubble)
-        if (
-          'streamOpenAiChatCompletion' in action ||
-          'webhookToExecute' in action
-        )
-          setIsSending(true)
-        const response = await executeClientSideAction({
-          clientSideAction: action,
-          context: {
-            apiHost: props.context.apiHost,
-            sessionId: props.initialChatReply.sessionId,
-          },
-          onMessageStream: streamMessage,
-        })
-        if (response && 'replyToSend' in response) {
-          sendMessage(response.replyToSend, response.logs)
-          return
-        }
-        if (response && 'blockedPopupUrl' in response)
-          setBlockedPopupUrl(response.blockedPopupUrl)
-      }
     })()
   })
 
@@ -210,27 +190,7 @@ export const ConversationContainer = (props: Props) => {
       const actionsBeforeFirstBubble = data.clientSideActions.filter((action) =>
         isNotDefined(action.lastBubbleBlockId)
       )
-      for (const action of actionsBeforeFirstBubble) {
+      processClientSideActions(actionsBeforeFirstBubble)
-        if (
-          'streamOpenAiChatCompletion' in action ||
-          'webhookToExecute' in action
-        )
-          setIsSending(true)
-        const response = await executeClientSideAction({
-          clientSideAction: action,
-          context: {
-            apiHost: props.context.apiHost,
-            sessionId: props.initialChatReply.sessionId,
-          },
-          onMessageStream: streamMessage,
-        })
-        if (response && 'replyToSend' in response) {
-          sendMessage(response.replyToSend, response.logs)
-          return
-        }
-        if (response && 'blockedPopupUrl' in response)
-          setBlockedPopupUrl(response.blockedPopupUrl)
-      }
     }
     setChatChunks((displayedChunks) => [
       ...displayedChunks,
@@ -267,27 +227,35 @@ export const ConversationContainer = (props: Props) => {
       const actionsToExecute = lastChunk.clientSideActions.filter(
         (action) => action.lastBubbleBlockId === blockId
       )
-      for (const action of actionsToExecute) {
+      await processClientSideActions(actionsToExecute)
-        if (
+    }
-          'streamOpenAiChatCompletion' in action ||
+  }
-          'webhookToExecute' in action
+
-        )
+  const processClientSideActions = async (
-          setIsSending(true)
+    actions: NonNullable<ContinueChatResponse['clientSideActions']>
-        const response = await executeClientSideAction({
+  ) => {
-          clientSideAction: action,
+    for (const action of actions) {
-          context: {
+      if (
-            apiHost: props.context.apiHost,
+        'streamOpenAiChatCompletion' in action ||
-            sessionId: props.initialChatReply.sessionId,
+        'webhookToExecute' in action ||
-          },
+        'stream' in action
-          onMessageStream: streamMessage,
+      )
-        })
+        setIsSending(true)
-        if (response && 'replyToSend' in response) {
+      const response = await executeClientSideAction({
-          sendMessage(response.replyToSend, response.logs)
+        clientSideAction: action,
-          return
+        context: {
-        }
+          apiHost: props.context.apiHost,
-        if (response && 'blockedPopupUrl' in response)
+          sessionId: props.initialChatReply.sessionId,
-          setBlockedPopupUrl(response.blockedPopupUrl)
+        },
+        onMessageStream: streamMessage,
+      })
+      if (response && 'replyToSend' in response) {
+        setIsSending(false)
+        sendMessage(response.replyToSend, response.logs)
+        return
       }
+      if (response && 'blockedPopupUrl' in response)
+        setBlockedPopupUrl(response.blockedPopupUrl)
     }
   }
 

packages/embeds/nextjs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/nextjs",
-  "version": "0.2.33",
+  "version": "0.2.34",
   "description": "Convenient library to display typebots on your Next.js website",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/embeds/react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typebot.io/react",
-  "version": "0.2.33",
+  "version": "0.2.34",
   "description": "Convenient library to display typebots on your React app",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/forge/blocks/openai/actions/askAssistant.tsx

@@ -24,6 +24,24 @@ export const askAssistant = createAction({
       label: 'Message',
       inputType: 'textarea',
     }),
+    functions: option
+      .array(
+        option.object({
+          name: option.string.layout({
+            fetcher: 'fetchAssistantFunctions',
+            label: 'Name',
+          }),
+          code: option.string.layout({
+            inputType: 'code',
+            label: 'Code',
+            lang: 'javascript',
+            moreInfoTooltip:
+              'A javascript code snippet that can use the defined parameters. It should return a value.',
+            withVariableButton: false,
+          }),
+        })
+      )
+      .layout({ accordion: 'Functions', itemLabel: 'function' }),
     responseMapping: option
       .saveResponseArray(['Message', 'Thread ID'] as const)
       .layout({
@@ -64,6 +82,40 @@ export const askAssistant = createAction({
       },
       dependencies: ['baseUrl', 'apiVersion'],
     },
+    {
+      id: 'fetchAssistantFunctions',
+      fetch: async ({ options, credentials }) => {
+        if (!options.assistantId) return []
+        const config = {
+          apiKey: credentials.apiKey,
+          baseURL: options.baseUrl,
+          defaultHeaders: {
+            'api-key': credentials.apiKey,
+          },
+          defaultQuery: options.apiVersion
+            ? {
+                'api-version': options.apiVersion,
+              }
+            : undefined,
+        } satisfies ClientOptions
+
+        const openai = new OpenAI(config)
+
+        const response = await openai.beta.assistants.retrieve(
+          options.assistantId
+        )
+
+        return response.tools
+          .filter((tool) => tool.type === 'function')
+          .map((tool) =>
+            tool.type === 'function' && tool.function.name
+              ? tool.function.name
+              : undefined
+          )
+          .filter(isDefined)
+      },
+      dependencies: ['baseUrl', 'apiVersion', 'assistantId'],
+    },
   ],
   getSetVariableIds: ({ responseMapping }) =>
     responseMapping?.map((r) => r.variableId).filter(isDefined) ?? [],
@@ -77,6 +129,7 @@ export const askAssistant = createAction({
         message,
         responseMapping,
         threadId,
+        functions,
       },
       variables,
       logs,
@@ -139,6 +192,45 @@ export const askAssistant = createAction({
         ) {
           throw new Error(run.status)
         }
+        if (run.status === 'requires_action') {
+          if (run.required_action?.type === 'submit_tool_outputs') {
+            const tool_outputs = (
+              await Promise.all(
+                run.required_action.submit_tool_outputs.tool_calls.map(
+                  async (toolCall) => {
+                    const parameters = JSON.parse(toolCall.function.arguments)
+
+                    const functionToExecute = functions?.find(
+                      (f) => f.name === toolCall.function.name
+                    )
+                    if (!functionToExecute) return
+
+                    const name = toolCall.function.name
+                    if (!name) return
+                    const func = AsyncFunction(
+                      ...Object.keys(parameters),
+                      functionToExecute.code
+                    )
+                    const output = await func(...Object.values(parameters))
+
+                    return {
+                      tool_call_id: toolCall.id,
+                      output,
+                    }
+                  }
+                )
+              )
+            ).filter(isDefined)
+
+            run = await openai.beta.threads.runs.submitToolOutputs(
+              currentThreadId,
+              run.id,
+              { tool_outputs }
+            )
+
+            await waitForRun(run)
+          }
+        }
       }
 
       await waitForRun(run)
@@ -170,3 +262,5 @@ export const askAssistant = createAction({
     },
   },
 })
+
+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor

packages/forge/blocks/openai/actions/createChatCompletion.tsx

@@ -1,11 +1,13 @@
 import { option, createAction } from '@typebot.io/forge'
 import OpenAI, { ClientOptions } from 'openai'
 import { defaultOpenAIOptions } from '../constants'
-import { OpenAIStream } from 'ai'
+import { OpenAIStream, ToolCallPayload } from 'ai'
 import { parseChatCompletionMessages } from '../helpers/parseChatCompletionMessages'
 import { isDefined } from '@typebot.io/lib'
 import { auth } from '../auth'
 import { baseOptions } from '../baseOptions'
+import { ChatCompletionTool } from 'openai/resources/chat/completions'
+import { parseToolParameters } from '../helpers/parseToolParameters'
 
 const nativeMessageContentSchema = {
   content: option.string.layout({
@@ -32,6 +34,48 @@ const assistantMessageItemSchema = option
   })
   .extend(nativeMessageContentSchema)
 
+export const parameterSchema = option.object({
+  name: option.string.layout({
+    label: 'Name',
+    placeholder: 'myVariable',
+    withVariableButton: false,
+  }),
+  type: option.enum(['string', 'number', 'boolean']).layout({
+    label: 'Type:',
+    direction: 'row',
+  }),
+  description: option.string.layout({
+    label: 'Description',
+    withVariableButton: false,
+  }),
+  required: option.boolean.layout({
+    label: 'Is required?',
+  }),
+})
+
+const functionToolItemSchema = option.object({
+  type: option.literal('function'),
+  name: option.string.layout({
+    label: 'Name',
+    placeholder: 'myFunctionName',
+    withVariableButton: false,
+  }),
+  description: option.string.layout({
+    label: 'Description',
+    placeholder: 'A brief description of what this function does.',
+    withVariableButton: false,
+  }),
+  parameters: option.array(parameterSchema).layout({ accordion: 'Parameters' }),
+  code: option.string.layout({
+    inputType: 'code',
+    label: 'Code',
+    lang: 'javascript',
+    moreInfoTooltip:
+      'A javascript code snippet that can use the defined parameters. It should return a value.',
+    withVariableButton: false,
+  }),
+})
+
 const dialogueMessageItemSchema = option.object({
   role: option.literal('Dialogue'),
   dialogueVariableId: option.string.layout({
@@ -61,6 +105,9 @@ export const options = option.object({
       ])
     )
     .layout({ accordion: 'Messages', itemLabel: 'message', isOrdered: true }),
+  tools: option
+    .array(option.discriminatedUnion('type', [functionToolItemSchema]))
+    .layout({ accordion: 'Tools', itemLabel: 'tool' }),
   temperature: option.number.layout({
     accordion: 'Advanced settings',
     label: 'Temperature',
@@ -131,20 +178,74 @@ export const createChatCompletion = createAction({
 
       const openai = new OpenAI(config)
 
+      const tools = options.tools
+        ? (options.tools
+            .filter((t) => t.name && t.parameters)
+            .map((t) => ({
+              type: 'function',
+              function: {
+                name: t.name as string,
+                description: t.description,
+                parameters: parseToolParameters(t.parameters!),
+              },
+            })) satisfies ChatCompletionTool[])
+        : undefined
+
+      const messages = parseChatCompletionMessages({ options, variables })
+
       const response = await openai.chat.completions.create({
         model: options.model ?? defaultOpenAIOptions.model,
         temperature: options.temperature
           ? Number(options.temperature)
           : undefined,
-        messages: parseChatCompletionMessages({ options, variables }),
+        messages,
+        tools,
       })
 
+      let message = response.choices[0].message
+      let totalTokens = response.usage?.total_tokens
+
+      if (message.tool_calls) {
+        messages.push(message)
+
+        for (const toolCall of message.tool_calls) {
+          const name = toolCall.function?.name
+          if (!name) continue
+          const toolDefinition = options.tools?.find((t) => t.name === name)
+          if (!toolDefinition?.code || !toolDefinition.parameters) continue
+          const func = AsyncFunction(
+            ...toolDefinition.parameters?.map((p) => p.name),
+            toolDefinition.code
+          )
+          const content = await func(
+            ...Object.values(JSON.parse(toolCall.function.arguments))
+          )
+          messages.push({
+            tool_call_id: toolCall.id,
+            role: 'tool',
+            content,
+          })
+        }
+
+        const secondResponse = await openai.chat.completions.create({
+          model: options.model ?? defaultOpenAIOptions.model,
+          temperature: options.temperature
+            ? Number(options.temperature)
+            : undefined,
+          messages,
+          tools,
+        })
+
+        message = secondResponse.choices[0].message
+        totalTokens = secondResponse.usage?.total_tokens
+      }
+
       options.responseMapping?.forEach((mapping) => {
         if (!mapping.variableId) return
         if (!mapping.item || mapping.item === 'Message content')
-          variables.set(mapping.variableId, response.choices[0].message.content)
+          variables.set(mapping.variableId, message.content)
         if (mapping.item === 'Total tokens')
-          variables.set(mapping.variableId, response.usage?.total_tokens)
+          variables.set(mapping.variableId, totalTokens)
       })
     },
     stream: {
@@ -168,17 +269,70 @@ export const createChatCompletion = createAction({
 
         const openai = new OpenAI(config)
 
+        const tools = options.tools
+          ? (options.tools
+              .filter((t) => t.name && t.parameters)
+              .map((t) => ({
+                type: 'function',
+                function: {
+                  name: t.name as string,
+                  description: t.description,
+                  parameters: parseToolParameters(t.parameters!),
+                },
+              })) satisfies ChatCompletionTool[])
+          : undefined
+
+        const messages = parseChatCompletionMessages({ options, variables })
+
         const response = await openai.chat.completions.create({
           model: options.model ?? defaultOpenAIOptions.model,
           temperature: options.temperature
             ? Number(options.temperature)
             : undefined,
           stream: true,
-          messages: parseChatCompletionMessages({ options, variables }),
+          messages,
+          tools,
         })
 
-        return OpenAIStream(response)
+        return OpenAIStream(response, {
+          experimental_onToolCall: async (
+            call: ToolCallPayload,
+            appendToolCallMessage
+          ) => {
+            for (const toolCall of call.tools) {
+              const name = toolCall.func?.name
+              if (!name) continue
+              const toolDefinition = options.tools?.find((t) => t.name === name)
+              if (!toolDefinition?.code || !toolDefinition.parameters) continue
+              const func = AsyncFunction(
+                ...toolDefinition.parameters?.map((p) => p.name),
+                toolDefinition.code
+              )
+              const content = await func(
+                ...Object.values(JSON.parse(toolCall.func.arguments as any))
+              )
+
+              const newMessages = appendToolCallMessage({
+                tool_call_id: toolCall.id,
+                function_name: toolCall.func.name,
+                tool_call_result: content,
+              })
+
+              return openai.chat.completions.create({
+                messages: [
+                  ...messages,
+                  ...newMessages,
+                ] as OpenAI.Chat.Completions.ChatCompletionMessageParam[],
+                model: options.model ?? defaultOpenAIOptions.model,
+                stream: true,
+                tools,
+              })
+            }
+          },
+        })
       },
     },
   },
 })
+
+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor

packages/forge/blocks/openai/helpers/parseToolParameters.ts

@@ -0,0 +1,22 @@
+import type { OpenAI } from 'openai'
+import { parameterSchema } from '../actions/createChatCompletion'
+import { z } from '@typebot.io/forge/zod'
+
+export const parseToolParameters = (
+  parameters: z.infer<typeof parameterSchema>[]
+): OpenAI.FunctionParameters => ({
+  type: 'object',
+  properties: parameters?.reduce<{
+    [x: string]: unknown
+  }>((acc, param) => {
+    if (!param.name) return acc
+    acc[param.name] = {
+      type: param.type,
+      description: param.description,
+    }
+    return acc
+  }, {}),
+  required:
+    parameters?.filter((param) => param.required).map((param) => param.name) ??
+    [],
+})

packages/forge/core/zod/extendWithTypebotLayout.ts

@@ -8,7 +8,8 @@ export interface ZodLayoutMetadata<
 > {
   accordion?: string
   label?: string
-  inputType?: 'variableDropdown' | 'textarea' | 'password'
+  inputType?: 'variableDropdown' | 'textarea' | 'password' | 'code'
+  lang?: 'javascript' | 'html' | 'css' | 'json'
   defaultValue?: T extends ZodDate ? string : TInferred
   placeholder?: string
   helperText?: string