From e58016e43a4dd58c809bcce604ab5af0ab91a41c Mon Sep 17 00:00:00 2001 From: Baptiste Arnaud Date: Tue, 2 May 2023 13:37:02 -0400 Subject: [PATCH] :zap: (openai) Truncate messages sequence automatically if reaching token limit --- .../templates/components/TemplatesModal.tsx | 2 + apps/builder/src/hooks/useToast.tsx | 2 +- apps/viewer/package.json | 11 ++-- .../openai/createChatCompletionOpenAI.ts | 60 +++++++++++++++---- .../features/blocks/integrations/openai.ts | 9 +++ pnpm-lock.yaml | 7 +++ 6 files changed, 73 insertions(+), 18 deletions(-) diff --git a/apps/builder/src/features/templates/components/TemplatesModal.tsx b/apps/builder/src/features/templates/components/TemplatesModal.tsx index 379056fd0..c7549f08c 100644 --- a/apps/builder/src/features/templates/components/TemplatesModal.tsx +++ b/apps/builder/src/features/templates/components/TemplatesModal.tsx @@ -77,6 +77,8 @@ export const TemplatesModal = ({ isOpen, onClose, onTypebotChoose }: Props) => { borderRightWidth={1} justify="space-between" flexShrink={0} + overflowY="scroll" + className="hide-scrollbar" > diff --git a/apps/builder/src/hooks/useToast.tsx b/apps/builder/src/hooks/useToast.tsx index 7fd205fb5..aa5be6103 100644 --- a/apps/builder/src/hooks/useToast.tsx +++ b/apps/builder/src/hooks/useToast.tsx @@ -17,7 +17,7 @@ export const useToast = () => { }: Omit) => { toast({ position: 'top-right', - duration: details ? null : undefined, + duration: details && status === 'error' ? null : undefined, render: ({ onClose }) => ( 0) newSessionState = await updateVariables(state)(variablesTransformedToList) @@ -148,7 +153,7 @@ export const createChatCompletionOpenAI = async ( } const parseMessages = - (variables: Variable[]) => + (variables: Variable[], model: ChatCompletionOpenAIOptions['model']) => ( messages: ChatCompletionOpenAIOptions['messages'] ): { @@ -156,8 +161,11 @@ const parseMessages = messages: ChatCompletionRequestMessage[] } => { const variablesTransformedToList: VariableWithValue[] = [] + const firstMessagesSequenceIndex = messages.findIndex( + (message) => message.role === 'Messages sequence ✨' + ) const parsedMessages = messages - .flatMap((message) => { + .flatMap((message, index) => { if (!message.role) return if (message.role === 'Messages sequence ✨') { if ( @@ -189,23 +197,51 @@ const parseMessages = variable.id === message.content?.assistantMessagesVariableId )?.value ?? []) as string[] + let allMessages: ChatCompletionRequestMessage[] = [] + if (userMessages.length > assistantMessages.length) - return userMessages.flatMap((userMessage, index) => [ + allMessages = userMessages.flatMap((userMessage, index) => [ { role: 'user', content: userMessage, }, - { role: 'assistant', content: assistantMessages[index] }, + { role: 'assistant', content: assistantMessages.at(index) ?? '' }, ]) satisfies ChatCompletionRequestMessage[] else { - return assistantMessages.flatMap((assistantMessage, index) => [ - { role: 'assistant', content: assistantMessage }, - { - role: 'user', - content: userMessages[index], - }, - ]) satisfies ChatCompletionRequestMessage[] + allMessages = assistantMessages.flatMap( + (assistantMessage, index) => [ + { role: 'assistant', content: assistantMessage }, + { + role: 'user', + content: userMessages.at(index) ?? '', + }, + ] + ) satisfies ChatCompletionRequestMessage[] } + + if (index !== firstMessagesSequenceIndex) return allMessages + + const encoder = encoding_for_model(model) + let messagesToSend: ChatCompletionRequestMessage[] = [] + let tokenCount = 0 + + for (let i = allMessages.length - 1; i >= 0; i--) { + const message = allMessages[i] + const tokens = encoder.encode(message.content) + + if ( + tokenCount + tokens.length - minTokenCompletion > + modelLimit[model] + ) { + break + } + tokenCount += tokens.length + messagesToSend = [message, ...messagesToSend] + } + + encoder.free() + + return messagesToSend } return { role: message.role, diff --git a/packages/schemas/features/blocks/integrations/openai.ts b/packages/schemas/features/blocks/integrations/openai.ts index c8df301f4..5d74fd4cb 100644 --- a/packages/schemas/features/blocks/integrations/openai.ts +++ b/packages/schemas/features/blocks/integrations/openai.ts @@ -14,6 +14,15 @@ export const chatCompletionModels = [ 'gpt-3.5-turbo-0301', ] as const +export const modelLimit = { + 'gpt-3.5-turbo': 4096, + 'gpt-3.5-turbo-0301': 4096, + 'gpt-4': 8192, + 'gpt-4-0314': 8192, + 'gpt-4-32k': 32768, + 'gpt-4-32k-0314': 32768, +} as const + export const chatCompletionMessageRoles = [ 'system', 'user', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4eb981713..7c389bcd1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -502,6 +502,9 @@ importers: apps/viewer: dependencies: + '@dqbd/tiktoken': + specifier: ^1.0.7 + version: 1.0.7 '@sentry/nextjs': specifier: 7.46.0 version: 7.46.0(next@13.2.4)(react@18.2.0) @@ -5228,6 +5231,10 @@ packages: - webpack-cli dev: false + /@dqbd/tiktoken@1.0.7: + resolution: {integrity: sha512-bhR5k5W+8GLzysjk8zTMVygQZsgvf7W1F0IlL4ZQ5ugjo5rCyiwGM5d8DYriXspytfu98tv59niang3/T+FoDw==} + dev: false + /@emotion/babel-plugin@11.10.6: resolution: {integrity: sha512-p2dAqtVrkhSa7xz1u/m9eHYdLi+en8NowrmXeF/dKtJpU8lCWli8RUAati7NcSl0afsBott48pdnANuD0wh9QQ==} dependencies: