blsmedia/bot
2
0
Fork 0
Code Activity

✨ (openai) Add tools and functions support (#1167)

Browse Source 
Closes #863

Got helped from #1162 for the implementation. Closing it in favor of
this PR.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Enhanced `CodeEditor` with additional properties for better form
control and validation.
- Introduced tools and functions in OpenAI integrations documentation
for custom JavaScript execution.
- Added capability to define and use custom JavaScript functions with
the OpenAI assistant.
- Expanded layout metadata options to include various input types and
languages.

- **Improvements**
- Updated the OpenAI actions to support new function execution features.

- **Documentation**
- Added new sections for tools and functions in the OpenAI integrations
guide.

- **Refactor**
- Refactored components and actions to integrate new features and
improve existing functionalities.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Baptiste Arnaud
2024-01-19 08:05:38 +01:00
committed by GitHub
parent 61bfe1bb96
commit f4d315fed5
12 changed files with 460 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
apps/builder/src/components/inputs/CodeEditor.tsx
View File

@ -1,11 +1,15 @@
import {
BoxProps,
Fade,
FormControl,
FormHelperText,
FormLabel,
HStack,
Stack,
useColorModeValue,
useDisclosure,
} from '@chakra-ui/react'
import { useEffect, useRef, useState } from 'react'
import React, { ReactNode, useEffect, useRef, useState } from 'react'
import { useDebouncedCallback } from 'use-debounce'
import { VariablesButton } from '@/features/variables/components/VariablesButton'
import { Variable } from '@typebot.io/schemas'
@ -16,8 +20,10 @@ import { githubLight } from '@uiw/codemirror-theme-github'
import { LanguageName, loadLanguage } from '@uiw/codemirror-extensions-langs'
import { isDefined } from '@udecode/plate-common'
import { CopyButton } from '../CopyButton'
import { MoreInfoTooltip } from '../MoreInfoTooltip'
type Props = {
label?: string
value?: string
defaultValue?: string
lang: LanguageName
@ -27,11 +33,18 @@ type Props = {
height?: string
maxHeight?: string
minWidth?: string
moreInfoTooltip?: string
helperText?: ReactNode
isRequired?: boolean
onChange?: (value: string) => void
}
export const CodeEditor = ({
label,
defaultValue,
lang,
moreInfoTooltip,
helperText,
isRequired,
onChange,
height = '250px',
maxHeight = '70vh',
@ -86,6 +99,21 @@ export const CodeEditor = ({
)
return (
<FormControl
isRequired={isRequired}
as={Stack}
justifyContent="space-between"
spacing={2}
flex="1"
>
{label && (
<FormLabel display="flex" flexShrink={0} gap="1" mb="0" mr="0">
{label}{' '}
{moreInfoTooltip && (
<MoreInfoTooltip>{moreInfoTooltip}</MoreInfoTooltip>
)}
</FormLabel>
)}
<HStack
align="flex-end"
spacing={0}
@ -137,7 +165,10 @@ export const CodeEditor = ({
}}
/>
{isVariableButtonDisplayed && (
<VariablesButton onSelectVariable={handleVariableSelected} size="sm" />
<VariablesButton
onSelectVariable={handleVariableSelected}
size="sm"
/>
)}
{isReadOnly && (
<Fade in={isOpen}>
@ -152,5 +183,7 @@ export const CodeEditor = ({
</Fade>
)}
</HStack>
{helperText && <FormHelperText mt="0">{helperText}</FormHelperText>}
</FormControl>
)
}

37
apps/builder/src/features/forge/components/zodLayouts/ZodFieldLayout.tsx
View File

@ -22,6 +22,7 @@ import { DropdownList } from '@/components/DropdownList'
import { ForgedBlockDefinition, ForgedBlock } from '@typebot.io/forge-schemas'
import { PrimitiveList } from '@/components/PrimitiveList'
import { SwitchWithLabel } from '@/components/inputs/SwitchWithLabel'
import { CodeEditor } from '@/components/inputs/CodeEditor'
const mdComponents = {
a: ({ href, children }) => (
@ -99,6 +100,8 @@ export const ZodFieldLayout = ({
<ZodArrayContent
data={data}
schema={schema}
blockDef={blockDef}
blockOptions={blockOptions}
layout={layout}
onDataChange={onDataChange}
isInAccordion
@ -111,6 +114,8 @@ export const ZodFieldLayout = ({
<ZodArrayContent
data={data}
schema={schema}
blockDef={blockDef}
blockOptions={blockOptions}
layout={layout}
onDataChange={onDataChange}
/>
@ -229,6 +234,28 @@ export const ZodFieldLayout = ({
/>
)
}
if (layout?.inputType === 'code')
return (
<CodeEditor
defaultValue={data ?? layout?.defaultValue}
lang={layout.lang ?? 'javascript'}
label={layout?.label}
placeholder={layout?.placeholder}
helperText={
layout?.helperText ? (
<Markdown components={mdComponents}>
{layout.helperText}
</Markdown>
) : undefined
}
isRequired={layout?.isRequired}
withVariableButton={layout?.withVariableButton}
moreInfoTooltip={layout.moreInfoTooltip}
onChange={onDataChange}
width={width}
/>
)
return (
<TextInput
defaultValue={data ?? layout?.defaultValue}
@ -254,12 +281,16 @@ export const ZodFieldLayout = ({
const ZodArrayContent = ({
schema,
data,
blockDef,
blockOptions,
layout,
isInAccordion,
onDataChange,
}: {
schema: z.ZodTypeAny
data: any
blockDef?: ForgedBlockDefinition
blockOptions?: ForgedBlock['options']
layout: ZodLayoutMetadata<ZodTypeAny> | undefined
isInAccordion?: boolean
onDataChange: (val: any) => void
@ -283,6 +314,8 @@ const ZodArrayContent = ({
<ZodFieldLayout
schema={schema._def.innerType._def.type}
data={item}
blockDef={blockDef}
blockOptions={blockOptions}
isInAccordion={isInAccordion}
onDataChange={onItemChange}
width="full"
@ -302,9 +335,11 @@ const ZodArrayContent = ({
isOrdered={layout?.isOrdered}
>
{({ item, onItemChange }) => (
<Stack p="4" rounded="md" flex="1" borderWidth="1px">
<Stack p="4" rounded="md" flex="1" borderWidth="1px" maxW="100%">
<ZodFieldLayout
schema={schema._def.innerType._def.type}
blockDef={blockDef}
blockOptions={blockOptions}
data={item}
isInAccordion={isInAccordion}
onDataChange={onItemChange}

16
apps/docs/editor/blocks/integrations/openai.mdx
View File

@ -33,6 +33,18 @@ Then you can give the OpenAI block access to this sequence of messages:
/>
</Frame>
### Tools
The tools section allows you to add functions that the OpenAI model can execute. Here is an example of a function named `getWeather` that returns 'Sunny and warm' if you ask about the weather of Paris and 'Rainy and cold' if you ask for any other city 😂.
A more useful example would be, of course, to call an API to get the weather of the city the user is asking about.
<Frame>
<img src="/images/blocks/integrations/openai/tools.png" alt="OpenAI tools" />
</Frame>
As you can see, the code block expects the body of the Javascript function. You can use the `return` keyword to return values.
## Ask assistant
This action allows you to talk with your [OpenAI assistant](https://platform.openai.com/assistants). All you have to do is to provide its ID.
@ -44,6 +56,10 @@ This action allows you to talk with your [OpenAI assistant](https://platform.ope
/>
</Frame>
### Functions
If you defined functions in your assistant, you can define the function to execute in the `Functions` section.
## Create speech
This action allows you to transform a text input into an audio URL that you can reuse in your bot.

BIN
apps/docs/images/blocks/integrations/openai/tools.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 171 KiB

2
packages/embeds/js/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@typebot.io/js",
"version": "0.2.33",
"version": "0.2.34",
"description": "Javascript library to display typebots on your website",
"type": "module",
"main": "dist/index.js",

58
packages/embeds/js/src/components/ConversationContainer/ConversationContainer.tsx
View File

@ -90,27 +90,7 @@ export const ConversationContainer = (props: Props) => {
const actionsBeforeFirstBubble = initialChunk.clientSideActions.filter(
(action) => isNotDefined(action.lastBubbleBlockId)
)
for (const action of actionsBeforeFirstBubble) {
if (
'streamOpenAiChatCompletion' in action ||
'webhookToExecute' in action
)
setIsSending(true)
const response = await executeClientSideAction({
clientSideAction: action,
context: {
apiHost: props.context.apiHost,
sessionId: props.initialChatReply.sessionId,
},
onMessageStream: streamMessage,
})
if (response && 'replyToSend' in response) {
sendMessage(response.replyToSend, response.logs)
return
}
if (response && 'blockedPopupUrl' in response)
setBlockedPopupUrl(response.blockedPopupUrl)
}
processClientSideActions(actionsBeforeFirstBubble)
})()
})
@ -210,27 +190,7 @@ export const ConversationContainer = (props: Props) => {
const actionsBeforeFirstBubble = data.clientSideActions.filter((action) =>
isNotDefined(action.lastBubbleBlockId)
)
for (const action of actionsBeforeFirstBubble) {
if (
'streamOpenAiChatCompletion' in action ||
'webhookToExecute' in action
)
setIsSending(true)
const response = await executeClientSideAction({
clientSideAction: action,
context: {
apiHost: props.context.apiHost,
sessionId: props.initialChatReply.sessionId,
},
onMessageStream: streamMessage,
})
if (response && 'replyToSend' in response) {
sendMessage(response.replyToSend, response.logs)
return
}
if (response && 'blockedPopupUrl' in response)
setBlockedPopupUrl(response.blockedPopupUrl)
}
processClientSideActions(actionsBeforeFirstBubble)
}
setChatChunks((displayedChunks) => [
...displayedChunks,
@ -267,10 +227,18 @@ export const ConversationContainer = (props: Props) => {
const actionsToExecute = lastChunk.clientSideActions.filter(
(action) => action.lastBubbleBlockId === blockId
)
for (const action of actionsToExecute) {
await processClientSideActions(actionsToExecute)
}
}
const processClientSideActions = async (
actions: NonNullable<ContinueChatResponse['clientSideActions']>
) => {
for (const action of actions) {
if (
'streamOpenAiChatCompletion' in action ||
'webhookToExecute' in action
'webhookToExecute' in action ||
'stream' in action
)
setIsSending(true)
const response = await executeClientSideAction({
@ -282,6 +250,7 @@ export const ConversationContainer = (props: Props) => {
onMessageStream: streamMessage,
})
if (response && 'replyToSend' in response) {
setIsSending(false)
sendMessage(response.replyToSend, response.logs)
return
}
@ -289,7 +258,6 @@ export const ConversationContainer = (props: Props) => {
setBlockedPopupUrl(response.blockedPopupUrl)
}
}
}
onCleanup(() => {
setStreamingMessage(undefined)

2
packages/embeds/nextjs/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@typebot.io/nextjs",
"version": "0.2.33",
"version": "0.2.34",
"description": "Convenient library to display typebots on your Next.js website",
"main": "dist/index.js",
"types": "dist/index.d.ts",

2
packages/embeds/react/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@typebot.io/react",
"version": "0.2.33",
"version": "0.2.34",
"description": "Convenient library to display typebots on your React app",
"main": "dist/index.js",
"types": "dist/index.d.ts",

94
packages/forge/blocks/openai/actions/askAssistant.tsx
View File

@ -24,6 +24,24 @@ export const askAssistant = createAction({
label: 'Message',
inputType: 'textarea',
}),
functions: option
.array(
option.object({
name: option.string.layout({
fetcher: 'fetchAssistantFunctions',
label: 'Name',
}),
code: option.string.layout({
inputType: 'code',
label: 'Code',
lang: 'javascript',
moreInfoTooltip:
'A javascript code snippet that can use the defined parameters. It should return a value.',
withVariableButton: false,
}),
})
)
.layout({ accordion: 'Functions', itemLabel: 'function' }),
responseMapping: option
.saveResponseArray(['Message', 'Thread ID'] as const)
.layout({
@ -64,6 +82,40 @@ export const askAssistant = createAction({
},
dependencies: ['baseUrl', 'apiVersion'],
},
{
id: 'fetchAssistantFunctions',
fetch: async ({ options, credentials }) => {
if (!options.assistantId) return []
const config = {
apiKey: credentials.apiKey,
baseURL: options.baseUrl,
defaultHeaders: {
'api-key': credentials.apiKey,
},
defaultQuery: options.apiVersion
? {
'api-version': options.apiVersion,
}
: undefined,
} satisfies ClientOptions
const openai = new OpenAI(config)
const response = await openai.beta.assistants.retrieve(
options.assistantId
)
return response.tools
.filter((tool) => tool.type === 'function')
.map((tool) =>
tool.type === 'function' && tool.function.name
? tool.function.name
: undefined
)
.filter(isDefined)
},
dependencies: ['baseUrl', 'apiVersion', 'assistantId'],
},
],
getSetVariableIds: ({ responseMapping }) =>
responseMapping?.map((r) => r.variableId).filter(isDefined) ?? [],
@ -77,6 +129,7 @@ export const askAssistant = createAction({
message,
responseMapping,
threadId,
functions,
},
variables,
logs,
@ -139,6 +192,45 @@ export const askAssistant = createAction({
) {
throw new Error(run.status)
}
if (run.status === 'requires_action') {
if (run.required_action?.type === 'submit_tool_outputs') {
const tool_outputs = (
await Promise.all(
run.required_action.submit_tool_outputs.tool_calls.map(
async (toolCall) => {
const parameters = JSON.parse(toolCall.function.arguments)
const functionToExecute = functions?.find(
(f) => f.name === toolCall.function.name
)
if (!functionToExecute) return
const name = toolCall.function.name
if (!name) return
const func = AsyncFunction(
...Object.keys(parameters),
functionToExecute.code
)
const output = await func(...Object.values(parameters))
return {
tool_call_id: toolCall.id,
output,
}
}
)
)
).filter(isDefined)
run = await openai.beta.threads.runs.submitToolOutputs(
currentThreadId,
run.id,
{ tool_outputs }
)
await waitForRun(run)
}
}
}
await waitForRun(run)
@ -170,3 +262,5 @@ export const askAssistant = createAction({
},
},
})
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor

166
packages/forge/blocks/openai/actions/createChatCompletion.tsx
View File

@ -1,11 +1,13 @@
import { option, createAction } from '@typebot.io/forge'
import OpenAI, { ClientOptions } from 'openai'
import { defaultOpenAIOptions } from '../constants'
import { OpenAIStream } from 'ai'
import { OpenAIStream, ToolCallPayload } from 'ai'
import { parseChatCompletionMessages } from '../helpers/parseChatCompletionMessages'
import { isDefined } from '@typebot.io/lib'
import { auth } from '../auth'
import { baseOptions } from '../baseOptions'
import { ChatCompletionTool } from 'openai/resources/chat/completions'
import { parseToolParameters } from '../helpers/parseToolParameters'
const nativeMessageContentSchema = {
content: option.string.layout({
@ -32,6 +34,48 @@ const assistantMessageItemSchema = option
})
.extend(nativeMessageContentSchema)
export const parameterSchema = option.object({
name: option.string.layout({
label: 'Name',
placeholder: 'myVariable',
withVariableButton: false,
}),
type: option.enum(['string', 'number', 'boolean']).layout({
label: 'Type:',
direction: 'row',
}),
description: option.string.layout({
label: 'Description',
withVariableButton: false,
}),
required: option.boolean.layout({
label: 'Is required?',
}),
})
const functionToolItemSchema = option.object({
type: option.literal('function'),
name: option.string.layout({
label: 'Name',
placeholder: 'myFunctionName',
withVariableButton: false,
}),
description: option.string.layout({
label: 'Description',
placeholder: 'A brief description of what this function does.',
withVariableButton: false,
}),
parameters: option.array(parameterSchema).layout({ accordion: 'Parameters' }),
code: option.string.layout({
inputType: 'code',
label: 'Code',
lang: 'javascript',
moreInfoTooltip:
'A javascript code snippet that can use the defined parameters. It should return a value.',
withVariableButton: false,
}),
})
const dialogueMessageItemSchema = option.object({
role: option.literal('Dialogue'),
dialogueVariableId: option.string.layout({
@ -61,6 +105,9 @@ export const options = option.object({
])
)
.layout({ accordion: 'Messages', itemLabel: 'message', isOrdered: true }),
tools: option
.array(option.discriminatedUnion('type', [functionToolItemSchema]))
.layout({ accordion: 'Tools', itemLabel: 'tool' }),
temperature: option.number.layout({
accordion: 'Advanced settings',
label: 'Temperature',
@ -131,20 +178,74 @@ export const createChatCompletion = createAction({
const openai = new OpenAI(config)
const tools = options.tools
? (options.tools
.filter((t) => t.name && t.parameters)
.map((t) => ({
type: 'function',
function: {
name: t.name as string,
description: t.description,
parameters: parseToolParameters(t.parameters!),
},
})) satisfies ChatCompletionTool[])
: undefined
const messages = parseChatCompletionMessages({ options, variables })
const response = await openai.chat.completions.create({
model: options.model ?? defaultOpenAIOptions.model,
temperature: options.temperature
? Number(options.temperature)
: undefined,
messages: parseChatCompletionMessages({ options, variables }),
messages,
tools,
})
let message = response.choices[0].message
let totalTokens = response.usage?.total_tokens
if (message.tool_calls) {
messages.push(message)
for (const toolCall of message.tool_calls) {
const name = toolCall.function?.name
if (!name) continue
const toolDefinition = options.tools?.find((t) => t.name === name)
if (!toolDefinition?.code || !toolDefinition.parameters) continue
const func = AsyncFunction(
...toolDefinition.parameters?.map((p) => p.name),
toolDefinition.code
)
const content = await func(
...Object.values(JSON.parse(toolCall.function.arguments))
)
messages.push({
tool_call_id: toolCall.id,
role: 'tool',
content,
})
}
const secondResponse = await openai.chat.completions.create({
model: options.model ?? defaultOpenAIOptions.model,
temperature: options.temperature
? Number(options.temperature)
: undefined,
messages,
tools,
})
message = secondResponse.choices[0].message
totalTokens = secondResponse.usage?.total_tokens
}
options.responseMapping?.forEach((mapping) => {
if (!mapping.variableId) return
if (!mapping.item || mapping.item === 'Message content')
variables.set(mapping.variableId, response.choices[0].message.content)
variables.set(mapping.variableId, message.content)
if (mapping.item === 'Total tokens')
variables.set(mapping.variableId, response.usage?.total_tokens)
variables.set(mapping.variableId, totalTokens)
})
},
stream: {
@ -168,17 +269,70 @@ export const createChatCompletion = createAction({
const openai = new OpenAI(config)
const tools = options.tools
? (options.tools
.filter((t) => t.name && t.parameters)
.map((t) => ({
type: 'function',
function: {
name: t.name as string,
description: t.description,
parameters: parseToolParameters(t.parameters!),
},
})) satisfies ChatCompletionTool[])
: undefined
const messages = parseChatCompletionMessages({ options, variables })
const response = await openai.chat.completions.create({
model: options.model ?? defaultOpenAIOptions.model,
temperature: options.temperature
? Number(options.temperature)
: undefined,
stream: true,
messages: parseChatCompletionMessages({ options, variables }),
messages,
tools,
})
return OpenAIStream(response)
return OpenAIStream(response, {
experimental_onToolCall: async (
call: ToolCallPayload,
appendToolCallMessage
) => {
for (const toolCall of call.tools) {
const name = toolCall.func?.name
if (!name) continue
const toolDefinition = options.tools?.find((t) => t.name === name)
if (!toolDefinition?.code || !toolDefinition.parameters) continue
const func = AsyncFunction(
...toolDefinition.parameters?.map((p) => p.name),
toolDefinition.code
)
const content = await func(
...Object.values(JSON.parse(toolCall.func.arguments as any))
)
const newMessages = appendToolCallMessage({
tool_call_id: toolCall.id,
function_name: toolCall.func.name,
tool_call_result: content,
})
return openai.chat.completions.create({
messages: [
...messages,
...newMessages,
] as OpenAI.Chat.Completions.ChatCompletionMessageParam[],
model: options.model ?? defaultOpenAIOptions.model,
stream: true,
tools,
})
}
},
})
},
},
},
})
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor

22
packages/forge/blocks/openai/helpers/parseToolParameters.ts Normal file
View File

@ -0,0 +1,22 @@
import type { OpenAI } from 'openai'
import { parameterSchema } from '../actions/createChatCompletion'
import { z } from '@typebot.io/forge/zod'
export const parseToolParameters = (
parameters: z.infer<typeof parameterSchema>[]
): OpenAI.FunctionParameters => ({
type: 'object',
properties: parameters?.reduce<{
[x: string]: unknown
}>((acc, param) => {
if (!param.name) return acc
acc[param.name] = {
type: param.type,
description: param.description,
}
return acc
}, {}),
required:
parameters?.filter((param) => param.required).map((param) => param.name) ??
[],
})

3
packages/forge/core/zod/extendWithTypebotLayout.ts
View File

@ -8,7 +8,8 @@ export interface ZodLayoutMetadata<
> {
accordion?: string
label?: string
inputType?: 'variableDropdown' | 'textarea' | 'password'
inputType?: 'variableDropdown' | 'textarea' | 'password' | 'code'
lang?: 'javascript' | 'html' | 'css' | 'json'
defaultValue?: T extends ZodDate ? string : TInferred
placeholder?: string
helperText?: string