✨ (openai) Add tools and functions support (#1167)
Closes #863 Got helped from #1162 for the implementation. Closing it in favor of this PR. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Enhanced `CodeEditor` with additional properties for better form control and validation. - Introduced tools and functions in OpenAI integrations documentation for custom JavaScript execution. - Added capability to define and use custom JavaScript functions with the OpenAI assistant. - Expanded layout metadata options to include various input types and languages. - **Improvements** - Updated the OpenAI actions to support new function execution features. - **Documentation** - Added new sections for tools and functions in the OpenAI integrations guide. - **Refactor** - Refactored components and actions to integrate new features and improve existing functionalities. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Baptiste Arnaud
GitHub
@ -24,6 +24,24 @@ export const askAssistant = createAction({
|
||||
label: 'Message',
|
||||
inputType: 'textarea',
|
||||
}),
|
||||
functions: option
|
||||
.array(
|
||||
option.object({
|
||||
name: option.string.layout({
|
||||
fetcher: 'fetchAssistantFunctions',
|
||||
label: 'Name',
|
||||
}),
|
||||
code: option.string.layout({
|
||||
inputType: 'code',
|
||||
label: 'Code',
|
||||
lang: 'javascript',
|
||||
moreInfoTooltip:
|
||||
'A javascript code snippet that can use the defined parameters. It should return a value.',
|
||||
withVariableButton: false,
|
||||
}),
|
||||
})
|
||||
)
|
||||
.layout({ accordion: 'Functions', itemLabel: 'function' }),
|
||||
responseMapping: option
|
||||
.saveResponseArray(['Message', 'Thread ID'] as const)
|
||||
.layout({
|
||||
@ -64,6 +82,40 @@ export const askAssistant = createAction({
|
||||
},
|
||||
dependencies: ['baseUrl', 'apiVersion'],
|
||||
},
|
||||
{
|
||||
id: 'fetchAssistantFunctions',
|
||||
fetch: async ({ options, credentials }) => {
|
||||
if (!options.assistantId) return []
|
||||
const config = {
|
||||
apiKey: credentials.apiKey,
|
||||
baseURL: options.baseUrl,
|
||||
defaultHeaders: {
|
||||
'api-key': credentials.apiKey,
|
||||
},
|
||||
defaultQuery: options.apiVersion
|
||||
? {
|
||||
'api-version': options.apiVersion,
|
||||
}
|
||||
: undefined,
|
||||
} satisfies ClientOptions
|
||||
|
||||
const openai = new OpenAI(config)
|
||||
|
||||
const response = await openai.beta.assistants.retrieve(
|
||||
options.assistantId
|
||||
)
|
||||
|
||||
return response.tools
|
||||
.filter((tool) => tool.type === 'function')
|
||||
.map((tool) =>
|
||||
tool.type === 'function' && tool.function.name
|
||||
? tool.function.name
|
||||
: undefined
|
||||
)
|
||||
.filter(isDefined)
|
||||
},
|
||||
dependencies: ['baseUrl', 'apiVersion', 'assistantId'],
|
||||
},
|
||||
],
|
||||
getSetVariableIds: ({ responseMapping }) =>
|
||||
responseMapping?.map((r) => r.variableId).filter(isDefined) ?? [],
|
||||
@ -77,6 +129,7 @@ export const askAssistant = createAction({
|
||||
message,
|
||||
responseMapping,
|
||||
threadId,
|
||||
functions,
|
||||
},
|
||||
variables,
|
||||
logs,
|
||||
@ -139,6 +192,45 @@ export const askAssistant = createAction({
|
||||
) {
|
||||
throw new Error(run.status)
|
||||
}
|
||||
if (run.status === 'requires_action') {
|
||||
if (run.required_action?.type === 'submit_tool_outputs') {
|
||||
const tool_outputs = (
|
||||
await Promise.all(
|
||||
run.required_action.submit_tool_outputs.tool_calls.map(
|
||||
async (toolCall) => {
|
||||
const parameters = JSON.parse(toolCall.function.arguments)
|
||||
|
||||
const functionToExecute = functions?.find(
|
||||
(f) => f.name === toolCall.function.name
|
||||
)
|
||||
if (!functionToExecute) return
|
||||
|
||||
const name = toolCall.function.name
|
||||
if (!name) return
|
||||
const func = AsyncFunction(
|
||||
...Object.keys(parameters),
|
||||
functionToExecute.code
|
||||
)
|
||||
const output = await func(...Object.values(parameters))
|
||||
|
||||
return {
|
||||
tool_call_id: toolCall.id,
|
||||
output,
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
).filter(isDefined)
|
||||
|
||||
run = await openai.beta.threads.runs.submitToolOutputs(
|
||||
currentThreadId,
|
||||
run.id,
|
||||
{ tool_outputs }
|
||||
)
|
||||
|
||||
await waitForRun(run)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await waitForRun(run)
|
||||
@ -170,3 +262,5 @@ export const askAssistant = createAction({
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
import { option, createAction } from '@typebot.io/forge'
|
||||
import OpenAI, { ClientOptions } from 'openai'
|
||||
import { defaultOpenAIOptions } from '../constants'
|
||||
import { OpenAIStream } from 'ai'
|
||||
import { OpenAIStream, ToolCallPayload } from 'ai'
|
||||
import { parseChatCompletionMessages } from '../helpers/parseChatCompletionMessages'
|
||||
import { isDefined } from '@typebot.io/lib'
|
||||
import { auth } from '../auth'
|
||||
import { baseOptions } from '../baseOptions'
|
||||
import { ChatCompletionTool } from 'openai/resources/chat/completions'
|
||||
import { parseToolParameters } from '../helpers/parseToolParameters'
|
||||
|
||||
const nativeMessageContentSchema = {
|
||||
content: option.string.layout({
|
||||
@ -32,6 +34,48 @@ const assistantMessageItemSchema = option
|
||||
})
|
||||
.extend(nativeMessageContentSchema)
|
||||
|
||||
export const parameterSchema = option.object({
|
||||
name: option.string.layout({
|
||||
label: 'Name',
|
||||
placeholder: 'myVariable',
|
||||
withVariableButton: false,
|
||||
}),
|
||||
type: option.enum(['string', 'number', 'boolean']).layout({
|
||||
label: 'Type:',
|
||||
direction: 'row',
|
||||
}),
|
||||
description: option.string.layout({
|
||||
label: 'Description',
|
||||
withVariableButton: false,
|
||||
}),
|
||||
required: option.boolean.layout({
|
||||
label: 'Is required?',
|
||||
}),
|
||||
})
|
||||
|
||||
const functionToolItemSchema = option.object({
|
||||
type: option.literal('function'),
|
||||
name: option.string.layout({
|
||||
label: 'Name',
|
||||
placeholder: 'myFunctionName',
|
||||
withVariableButton: false,
|
||||
}),
|
||||
description: option.string.layout({
|
||||
label: 'Description',
|
||||
placeholder: 'A brief description of what this function does.',
|
||||
withVariableButton: false,
|
||||
}),
|
||||
parameters: option.array(parameterSchema).layout({ accordion: 'Parameters' }),
|
||||
code: option.string.layout({
|
||||
inputType: 'code',
|
||||
label: 'Code',
|
||||
lang: 'javascript',
|
||||
moreInfoTooltip:
|
||||
'A javascript code snippet that can use the defined parameters. It should return a value.',
|
||||
withVariableButton: false,
|
||||
}),
|
||||
})
|
||||
|
||||
const dialogueMessageItemSchema = option.object({
|
||||
role: option.literal('Dialogue'),
|
||||
dialogueVariableId: option.string.layout({
|
||||
@ -61,6 +105,9 @@ export const options = option.object({
|
||||
])
|
||||
)
|
||||
.layout({ accordion: 'Messages', itemLabel: 'message', isOrdered: true }),
|
||||
tools: option
|
||||
.array(option.discriminatedUnion('type', [functionToolItemSchema]))
|
||||
.layout({ accordion: 'Tools', itemLabel: 'tool' }),
|
||||
temperature: option.number.layout({
|
||||
accordion: 'Advanced settings',
|
||||
label: 'Temperature',
|
||||
@ -131,20 +178,74 @@ export const createChatCompletion = createAction({
|
||||
|
||||
const openai = new OpenAI(config)
|
||||
|
||||
const tools = options.tools
|
||||
? (options.tools
|
||||
.filter((t) => t.name && t.parameters)
|
||||
.map((t) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: t.name as string,
|
||||
description: t.description,
|
||||
parameters: parseToolParameters(t.parameters!),
|
||||
},
|
||||
})) satisfies ChatCompletionTool[])
|
||||
: undefined
|
||||
|
||||
const messages = parseChatCompletionMessages({ options, variables })
|
||||
|
||||
const response = await openai.chat.completions.create({
|
||||
model: options.model ?? defaultOpenAIOptions.model,
|
||||
temperature: options.temperature
|
||||
? Number(options.temperature)
|
||||
: undefined,
|
||||
messages: parseChatCompletionMessages({ options, variables }),
|
||||
messages,
|
||||
tools,
|
||||
})
|
||||
|
||||
let message = response.choices[0].message
|
||||
let totalTokens = response.usage?.total_tokens
|
||||
|
||||
if (message.tool_calls) {
|
||||
messages.push(message)
|
||||
|
||||
for (const toolCall of message.tool_calls) {
|
||||
const name = toolCall.function?.name
|
||||
if (!name) continue
|
||||
const toolDefinition = options.tools?.find((t) => t.name === name)
|
||||
if (!toolDefinition?.code || !toolDefinition.parameters) continue
|
||||
const func = AsyncFunction(
|
||||
...toolDefinition.parameters?.map((p) => p.name),
|
||||
toolDefinition.code
|
||||
)
|
||||
const content = await func(
|
||||
...Object.values(JSON.parse(toolCall.function.arguments))
|
||||
)
|
||||
messages.push({
|
||||
tool_call_id: toolCall.id,
|
||||
role: 'tool',
|
||||
content,
|
||||
})
|
||||
}
|
||||
|
||||
const secondResponse = await openai.chat.completions.create({
|
||||
model: options.model ?? defaultOpenAIOptions.model,
|
||||
temperature: options.temperature
|
||||
? Number(options.temperature)
|
||||
: undefined,
|
||||
messages,
|
||||
tools,
|
||||
})
|
||||
|
||||
message = secondResponse.choices[0].message
|
||||
totalTokens = secondResponse.usage?.total_tokens
|
||||
}
|
||||
|
||||
options.responseMapping?.forEach((mapping) => {
|
||||
if (!mapping.variableId) return
|
||||
if (!mapping.item || mapping.item === 'Message content')
|
||||
variables.set(mapping.variableId, response.choices[0].message.content)
|
||||
variables.set(mapping.variableId, message.content)
|
||||
if (mapping.item === 'Total tokens')
|
||||
variables.set(mapping.variableId, response.usage?.total_tokens)
|
||||
variables.set(mapping.variableId, totalTokens)
|
||||
})
|
||||
},
|
||||
stream: {
|
||||
@ -168,17 +269,70 @@ export const createChatCompletion = createAction({
|
||||
|
||||
const openai = new OpenAI(config)
|
||||
|
||||
const tools = options.tools
|
||||
? (options.tools
|
||||
.filter((t) => t.name && t.parameters)
|
||||
.map((t) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: t.name as string,
|
||||
description: t.description,
|
||||
parameters: parseToolParameters(t.parameters!),
|
||||
},
|
||||
})) satisfies ChatCompletionTool[])
|
||||
: undefined
|
||||
|
||||
const messages = parseChatCompletionMessages({ options, variables })
|
||||
|
||||
const response = await openai.chat.completions.create({
|
||||
model: options.model ?? defaultOpenAIOptions.model,
|
||||
temperature: options.temperature
|
||||
? Number(options.temperature)
|
||||
: undefined,
|
||||
stream: true,
|
||||
messages: parseChatCompletionMessages({ options, variables }),
|
||||
messages,
|
||||
tools,
|
||||
})
|
||||
|
||||
return OpenAIStream(response)
|
||||
return OpenAIStream(response, {
|
||||
experimental_onToolCall: async (
|
||||
call: ToolCallPayload,
|
||||
appendToolCallMessage
|
||||
) => {
|
||||
for (const toolCall of call.tools) {
|
||||
const name = toolCall.func?.name
|
||||
if (!name) continue
|
||||
const toolDefinition = options.tools?.find((t) => t.name === name)
|
||||
if (!toolDefinition?.code || !toolDefinition.parameters) continue
|
||||
const func = AsyncFunction(
|
||||
...toolDefinition.parameters?.map((p) => p.name),
|
||||
toolDefinition.code
|
||||
)
|
||||
const content = await func(
|
||||
...Object.values(JSON.parse(toolCall.func.arguments as any))
|
||||
)
|
||||
|
||||
const newMessages = appendToolCallMessage({
|
||||
tool_call_id: toolCall.id,
|
||||
function_name: toolCall.func.name,
|
||||
tool_call_result: content,
|
||||
})
|
||||
|
||||
return openai.chat.completions.create({
|
||||
messages: [
|
||||
...messages,
|
||||
...newMessages,
|
||||
] as OpenAI.Chat.Completions.ChatCompletionMessageParam[],
|
||||
model: options.model ?? defaultOpenAIOptions.model,
|
||||
stream: true,
|
||||
tools,
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor
|
||||
|
||||
22
packages/forge/blocks/openai/helpers/parseToolParameters.ts
Normal file
22
packages/forge/blocks/openai/helpers/parseToolParameters.ts
Normal file
@ -0,0 +1,22 @@
|
||||
import type { OpenAI } from 'openai'
|
||||
import { parameterSchema } from '../actions/createChatCompletion'
|
||||
import { z } from '@typebot.io/forge/zod'
|
||||
|
||||
export const parseToolParameters = (
|
||||
parameters: z.infer<typeof parameterSchema>[]
|
||||
): OpenAI.FunctionParameters => ({
|
||||
type: 'object',
|
||||
properties: parameters?.reduce<{
|
||||
[x: string]: unknown
|
||||
}>((acc, param) => {
|
||||
if (!param.name) return acc
|
||||
acc[param.name] = {
|
||||
type: param.type,
|
||||
description: param.description,
|
||||
}
|
||||
return acc
|
||||
}, {}),
|
||||
required:
|
||||
parameters?.filter((param) => param.required).map((param) => param.name) ??
|
||||
[],
|
||||
})
|
||||
@ -8,7 +8,8 @@ export interface ZodLayoutMetadata<
|
||||
> {
|
||||
accordion?: string
|
||||
label?: string
|
||||
inputType?: 'variableDropdown' | 'textarea' | 'password'
|
||||
inputType?: 'variableDropdown' | 'textarea' | 'password' | 'code'
|
||||
lang?: 'javascript' | 'html' | 'css' | 'json'
|
||||
defaultValue?: T extends ZodDate ? string : TInferred
|
||||
placeholder?: string
|
||||
helperText?: string
|
||||
|
||||
Reference in New Issue
Block a user