⚡ (openai) Add new models and remove tiktoken

Instead of computing total tokens with tiktoken we just attempt retries after trimming the first message
2023-06-16 16:50:23 +02:00
parent e54aab452a
commit 83f2a29faa
11 changed files with 331 additions and 245 deletions
--- a/apps/viewer/src/features/blocks/integrations/openai/createChatCompletionOpenAI.ts
+++ b/apps/viewer/src/features/blocks/integrations/openai/createChatCompletionOpenAI.ts
@@ -1,36 +1,19 @@
 import { ExecuteIntegrationResponse } from '@/features/chat/types'
-import { transformStringVariablesToList } from '@/features/variables/transformVariablesToList'
 import prisma from '@/lib/prisma'
-import {
-  ChatReply,
-  SessionState,
-  Variable,
-  VariableWithValue,
-} from '@typebot.io/schemas'
+import { SessionState } from '@typebot.io/schemas'
 import {
  ChatCompletionOpenAIOptions,
  OpenAICredentials,
-  modelLimit,
 } from '@typebot.io/schemas/features/blocks/integrations/openai'
-import type {
-  ChatCompletionRequestMessage,
-  CreateChatCompletionRequest,
-  CreateChatCompletionResponse,
-} from 'openai'
-import { byId, isNotEmpty, isEmpty } from '@typebot.io/lib'
+import { isEmpty } from '@typebot.io/lib'
 import { decrypt, isCredentialsV2 } from '@typebot.io/lib/api/encryption'
-import { saveErrorLog } from '@/features/logs/saveErrorLog'
 import { updateVariables } from '@/features/variables/updateVariables'
-import { parseVariables } from '@/features/variables/parseVariables'
 import { parseVariableNumber } from '@/features/variables/parseVariableNumber'
-import { encoding_for_model } from '@dqbd/tiktoken'
-import got from 'got'
 import { resumeChatCompletion } from './resumeChatCompletion'
 import { isPlaneteScale } from '@/helpers/api/isPlanetScale'
 import { isVercel } from '@/helpers/api/isVercel'
-
-const minTokenCompletion = 200
-const createChatEndpoint = 'https://api.openai.com/v1/chat/completions'
+import { parseChatCompletionMessages } from './parseChatCompletionMessages'
+import { executeChatCompletionOpenAIRequest } from './executeChatCompletionOpenAIRequest'

 export const createChatCompletionOpenAI = async (
  state: SessionState,
@@ -63,9 +46,8 @@ export const createChatCompletionOpenAI = async (
    credentials.data,
    credentials.iv
  )) as OpenAICredentials['data']
-  const { variablesTransformedToList, messages } = parseMessages(
-    newSessionState.typebot.variables,
-    options.model
+  const { variablesTransformedToList, messages } = parseChatCompletionMessages(
+    newSessionState.typebot.variables
  )(options.messages)
  if (variablesTransformedToList.length > 0)
    newSessionState = await updateVariables(state)(variablesTransformedToList)
@@ -74,177 +56,37 @@ export const createChatCompletionOpenAI = async (
    options.advancedSettings?.temperature
  )

-  try {
-    if (
-      isPlaneteScale() &&
-      isVercel() &&
-      isCredentialsV2(credentials) &&
-      newSessionState.isStreamEnabled
-    )
-      return {
-        clientSideActions: [{ streamOpenAiChatCompletion: { messages } }],
-        outgoingEdgeId,
-        newSessionState,
-      }
-    const response = await got
-      .post(createChatEndpoint, {
-        headers: {
-          Authorization: `Bearer ${apiKey}`,
-        },
-        json: {
-          model: options.model,
-          messages,
-          temperature,
-        } satisfies CreateChatCompletionRequest,
-      })
-      .json<CreateChatCompletionResponse>()
-    const messageContent = response.choices.at(0)?.message?.content
-    const totalTokens = response.usage?.total_tokens
-    if (isEmpty(messageContent)) {
-      console.error('OpenAI block returned empty message', response)
-      return { outgoingEdgeId, newSessionState }
-    }
-    return resumeChatCompletion(newSessionState, {
-      options,
-      outgoingEdgeId,
-    })(messageContent, totalTokens)
-  } catch (err) {
-    const log: NonNullable<ChatReply['logs']>[number] = {
-      status: 'error',
-      description: 'OpenAI block returned error',
-    }
-
-    if (err && typeof err === 'object') {
-      if ('response' in err) {
-        const { status, data } = err.response as {
-          status: string
-          data: string
-        }
-        log.details = {
-          status,
-          data,
-        }
-      } else if ('message' in err) {
-        log.details = err.message
-      }
-    }
-
-    state.result &&
-      (await saveErrorLog({
-        resultId: state.result.id,
-        message: log.description,
-        details: log.details,
-      }))
+  if (
+    isPlaneteScale() &&
+    isVercel() &&
+    isCredentialsV2(credentials) &&
+    newSessionState.isStreamEnabled
+  )
    return {
+      clientSideActions: [{ streamOpenAiChatCompletion: { messages } }],
      outgoingEdgeId,
-      logs: [log],
      newSessionState,
    }
-  }
-}
-
-const parseMessages =
-  (variables: Variable[], model: ChatCompletionOpenAIOptions['model']) =>
-  (
-    messages: ChatCompletionOpenAIOptions['messages']
-  ): {
-    variablesTransformedToList: VariableWithValue[]
-    messages: ChatCompletionRequestMessage[]
-  } => {
-    const variablesTransformedToList: VariableWithValue[] = []
-    const firstMessagesSequenceIndex = messages.findIndex(
-      (message) => message.role === 'Messages sequence ✨'
-    )
-    const parsedMessages = messages
-      .flatMap((message, index) => {
-        if (!message.role) return
-        if (message.role === 'Messages sequence ✨') {
-          if (
-            !message.content?.assistantMessagesVariableId ||
-            !message.content?.userMessagesVariableId
-          )
-            return
-          variablesTransformedToList.push(
-            ...transformStringVariablesToList(variables)([
-              message.content.assistantMessagesVariableId,
-              message.content.userMessagesVariableId,
-            ])
-          )
-          const updatedVariables = variables.map((variable) => {
-            const variableTransformedToList = variablesTransformedToList.find(
-              byId(variable.id)
-            )
-            if (variableTransformedToList) return variableTransformedToList
-            return variable
-          })
-
-          const userMessages = (updatedVariables.find(
-            (variable) =>
-              variable.id === message.content?.userMessagesVariableId
-          )?.value ?? []) as string[]
-
-          const assistantMessages = (updatedVariables.find(
-            (variable) =>
-              variable.id === message.content?.assistantMessagesVariableId
-          )?.value ?? []) as string[]
-
-          let allMessages: ChatCompletionRequestMessage[] = []
-
-          if (userMessages.length > assistantMessages.length)
-            allMessages = userMessages.flatMap((userMessage, index) => [
-              {
-                role: 'user',
-                content: userMessage,
-              },
-              { role: 'assistant', content: assistantMessages.at(index) ?? '' },
-            ]) satisfies ChatCompletionRequestMessage[]
-          else {
-            allMessages = assistantMessages.flatMap(
-              (assistantMessage, index) => [
-                { role: 'assistant', content: assistantMessage },
-                {
-                  role: 'user',
-                  content: userMessages.at(index) ?? '',
-                },
-              ]
-            ) satisfies ChatCompletionRequestMessage[]
-          }
-
-          if (index !== firstMessagesSequenceIndex) return allMessages
-
-          const encoder = encoding_for_model(model)
-          let messagesToSend: ChatCompletionRequestMessage[] = []
-          let tokenCount = 0
-
-          for (let i = allMessages.length - 1; i >= 0; i--) {
-            const message = allMessages[i]
-            const tokens = encoder.encode(message.content)
-
-            if (
-              tokenCount + tokens.length - minTokenCompletion >
-              modelLimit[model]
-            ) {
-              break
-            }
-            tokenCount += tokens.length
-            messagesToSend = [message, ...messagesToSend]
-          }
-
-          encoder.free()
-
-          return messagesToSend
-        }
-        return {
-          role: message.role,
-          content: parseVariables(variables)(message.content),
-        } satisfies ChatCompletionRequestMessage
-      })
-      .filter(
-        (message) => isNotEmpty(message?.role) && isNotEmpty(message?.content)
-      ) as ChatCompletionRequestMessage[]
-
+  const { response, logs } = await executeChatCompletionOpenAIRequest({
+    apiKey,
+    messages,
+    model: options.model,
+    temperature,
+  })
+  if (!response)
    return {
-      variablesTransformedToList,
-      messages: parsedMessages,
+      outgoingEdgeId,
+      logs,
    }
+  const messageContent = response.choices.at(0)?.message?.content
+  const totalTokens = response.usage?.total_tokens
+  if (isEmpty(messageContent)) {
+    console.error('OpenAI block returned empty message', response)
+    return { outgoingEdgeId, newSessionState }
  }
+  return resumeChatCompletion(newSessionState, {
+    options,
+    outgoingEdgeId,
+    logs,
+  })(messageContent, totalTokens)
+}
--- a/apps/viewer/src/features/blocks/integrations/openai/executeChatCompletionOpenAIRequest.ts
+++ b/apps/viewer/src/features/blocks/integrations/openai/executeChatCompletionOpenAIRequest.ts
@@ -0,0 +1,74 @@
+import { ChatReply } from '@typebot.io/schemas'
+import got, { HTTPError } from 'got'
+import type {
+  CreateChatCompletionRequest,
+  CreateChatCompletionResponse,
+} from 'openai'
+
+const createChatEndpoint = 'https://api.openai.com/v1/chat/completions'
+
+type Props = Pick<CreateChatCompletionRequest, 'messages' | 'model'> & {
+  apiKey: string
+  temperature: number | undefined
+  currentLogs?: ChatReply['logs']
+  isRetrying?: boolean
+}
+
+export const executeChatCompletionOpenAIRequest = async ({
+  apiKey,
+  model,
+  messages,
+  temperature,
+  currentLogs = [],
+}: Props): Promise<{
+  response?: CreateChatCompletionResponse
+  logs?: ChatReply['logs']
+}> => {
+  const logs: ChatReply['logs'] = currentLogs
+  if (messages.length === 0) return { logs }
+  try {
+    const response = await got
+      .post(createChatEndpoint, {
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+        },
+        json: {
+          model,
+          messages,
+          temperature,
+        } satisfies CreateChatCompletionRequest,
+      })
+      .json<CreateChatCompletionResponse>()
+    return { response, logs }
+  } catch (error) {
+    if (error instanceof HTTPError) {
+      if (error.response.statusCode === 400) {
+        const log = {
+          status: 'info',
+          description:
+            'Max tokens limit reached, automatically trimming first message.',
+        }
+        logs.push(log)
+
+        return executeChatCompletionOpenAIRequest({
+          apiKey,
+          model,
+          messages: messages.slice(1),
+          temperature,
+          currentLogs: logs,
+        })
+      }
+      logs.push({
+        status: 'error',
+        description: `OpenAI API error - ${error.response.statusCode}`,
+        details: error.response.body,
+      })
+      return { logs }
+    }
+    logs.push({
+      status: 'error',
+      description: `Internal error`,
+    })
+    return { logs }
+  }
+}
--- a/apps/viewer/src/features/blocks/integrations/openai/parseChatCompletionMessages.ts
+++ b/apps/viewer/src/features/blocks/integrations/openai/parseChatCompletionMessages.ts
@@ -0,0 +1,90 @@
+import { parseVariables } from '@/features/variables/parseVariables'
+import { transformStringVariablesToList } from '@/features/variables/transformVariablesToList'
+import { byId, isNotEmpty } from '@typebot.io/lib'
+import { Variable, VariableWithValue } from '@typebot.io/schemas'
+import { ChatCompletionOpenAIOptions } from '@typebot.io/schemas/features/blocks/integrations/openai'
+import type { ChatCompletionRequestMessage } from 'openai'
+
+export const parseChatCompletionMessages =
+  (variables: Variable[]) =>
+  (
+    messages: ChatCompletionOpenAIOptions['messages']
+  ): {
+    variablesTransformedToList: VariableWithValue[]
+    messages: ChatCompletionRequestMessage[]
+  } => {
+    const variablesTransformedToList: VariableWithValue[] = []
+    const parsedMessages = messages
+      .flatMap((message) => {
+        if (!message.role) return
+        if (message.role === 'Messages sequence ✨') {
+          if (
+            !message.content?.assistantMessagesVariableId ||
+            !message.content?.userMessagesVariableId
+          )
+            return
+          variablesTransformedToList.push(
+            ...transformStringVariablesToList(variables)([
+              message.content.assistantMessagesVariableId,
+              message.content.userMessagesVariableId,
+            ])
+          )
+          const updatedVariables = variables.map((variable) => {
+            const variableTransformedToList = variablesTransformedToList.find(
+              byId(variable.id)
+            )
+            if (variableTransformedToList) return variableTransformedToList
+            return variable
+          })
+
+          const userMessages = (updatedVariables.find(
+            (variable) =>
+              variable.id === message.content?.userMessagesVariableId
+          )?.value ?? []) as string[]
+
+          const assistantMessages = (updatedVariables.find(
+            (variable) =>
+              variable.id === message.content?.assistantMessagesVariableId
+          )?.value ?? []) as string[]
+
+          let allMessages: ChatCompletionRequestMessage[] = []
+
+          if (userMessages.length > assistantMessages.length)
+            allMessages = userMessages.flatMap((userMessage, index) => [
+              {
+                role: 'user',
+                content: userMessage,
+              },
+              { role: 'assistant', content: assistantMessages.at(index) ?? '' },
+            ]) satisfies ChatCompletionRequestMessage[]
+          else {
+            allMessages = assistantMessages.flatMap(
+              (assistantMessage, index) => [
+                { role: 'assistant', content: assistantMessage },
+                {
+                  role: 'user',
+                  content: userMessages.at(index) ?? '',
+                },
+              ]
+            ) satisfies ChatCompletionRequestMessage[]
+          }
+
+          return allMessages
+        }
+        return {
+          role: message.role,
+          content: parseVariables(variables)(message.content),
+          name: message.name
+            ? parseVariables(variables)(message.name)
+            : undefined,
+        } satisfies ChatCompletionRequestMessage
+      })
+      .filter(
+        (message) => isNotEmpty(message?.role) && isNotEmpty(message?.content)
+      ) as ChatCompletionRequestMessage[]
+
+    return {
+      variablesTransformedToList,
+      messages: parsedMessages,
+    }
+  }
--- a/apps/viewer/src/features/blocks/integrations/openai/resumeChatCompletion.ts
+++ b/apps/viewer/src/features/blocks/integrations/openai/resumeChatCompletion.ts
@@ -1,7 +1,7 @@
 import { saveSuccessLog } from '@/features/logs/saveSuccessLog'
 import { updateVariables } from '@/features/variables/updateVariables'
 import { byId, isDefined } from '@typebot.io/lib'
-import { SessionState } from '@typebot.io/schemas'
+import { ChatReply, SessionState } from '@typebot.io/schemas'
 import { ChatCompletionOpenAIOptions } from '@typebot.io/schemas/features/blocks/integrations/openai'
 import { VariableWithUnknowValue } from '@typebot.io/schemas/features/typebot/variable'

@@ -11,7 +11,12 @@ export const resumeChatCompletion =
    {
      outgoingEdgeId,
      options,
-    }: { outgoingEdgeId?: string; options: ChatCompletionOpenAIOptions }
+      logs,
+    }: {
+      outgoingEdgeId?: string
+      options: ChatCompletionOpenAIOptions
+      logs?: ChatReply['logs']
+    }
  ) =>
  async (message: string, totalTokens?: number) => {
    let newSessionState = state
@@ -48,5 +53,6 @@ export const resumeChatCompletion =
    return {
      outgoingEdgeId,
      newSessionState,
+      logs,
    }
  }