✨ (openai) Add create speech OpenAI action

Closes #1025

packages/bot-engine/blocks/integrations/openai/audio/createSpeechOpenAI.ts

@@ -0,0 +1,121 @@
+import { SessionState } from '@typebot.io/schemas'
+import {
+  CreateSpeechOpenAIOptions,
+  OpenAICredentials,
+} from '@typebot.io/schemas/features/blocks/integrations/openai'
+import { isNotEmpty } from '@typebot.io/lib'
+import { decrypt } from '@typebot.io/lib/api/encryption/decrypt'
+import prisma from '@typebot.io/lib/prisma'
+import { defaultOpenAIOptions } from '@typebot.io/schemas/features/blocks/integrations/openai/constants'
+import { ExecuteIntegrationResponse } from '../../../../types'
+import OpenAI, { ClientOptions } from 'openai'
+import { uploadFileToBucket } from '@typebot.io/lib/s3/uploadFileToBucket'
+import { updateVariablesInSession } from '../../../../variables/updateVariablesInSession'
+import { createId } from '@paralleldrive/cuid2'
+import { parseVariables } from '../../../../variables/parseVariables'
+
+export const createSpeechOpenAI = async (
+  state: SessionState,
+  {
+    outgoingEdgeId,
+    options,
+  }: {
+    outgoingEdgeId?: string
+    options: CreateSpeechOpenAIOptions
+  }
+): Promise<ExecuteIntegrationResponse> => {
+  let newSessionState = state
+  const noCredentialsError = {
+    status: 'error',
+    description: 'Make sure to select an OpenAI account',
+  }
+
+  if (!options.input || !options.voice || !options.saveUrlInVariableId) {
+    return {
+      outgoingEdgeId,
+      logs: [
+        {
+          status: 'error',
+          description:
+            'Make sure to enter an input, select a voice and select a variable to save the URL in',
+        },
+      ],
+    }
+  }
+
+  if (!options.credentialsId) {
+    return {
+      outgoingEdgeId,
+      logs: [noCredentialsError],
+    }
+  }
+  const credentials = await prisma.credentials.findUnique({
+    where: {
+      id: options.credentialsId,
+    },
+  })
+  if (!credentials) {
+    console.error('Could not find credentials in database')
+    return { outgoingEdgeId, logs: [noCredentialsError] }
+  }
+  const { apiKey } = (await decrypt(
+    credentials.data,
+    credentials.iv
+  )) as OpenAICredentials['data']
+
+  const config = {
+    apiKey,
+    baseURL: options.baseUrl ?? defaultOpenAIOptions.baseUrl,
+    defaultHeaders: {
+      'api-key': apiKey,
+    },
+    defaultQuery: isNotEmpty(options.apiVersion)
+      ? {
+          'api-version': options.apiVersion,
+        }
+      : undefined,
+  } satisfies ClientOptions
+
+  const openai = new OpenAI(config)
+
+  const variables = newSessionState.typebotsQueue[0].typebot.variables
+  const saveUrlInVariable = variables.find(
+    (v) => v.id === options.saveUrlInVariableId
+  )
+
+  if (!saveUrlInVariable) {
+    return {
+      outgoingEdgeId,
+      logs: [
+        {
+          status: 'error',
+          description: 'Could not find variable to save URL in',
+        },
+      ],
+    }
+  }
+
+  const rawAudio = (await openai.audio.speech.create({
+    input: parseVariables(variables)(options.input),
+    voice: options.voice,
+    model: options.model as 'tts-1' | 'tts-1-hd',
+  })) as any
+
+  const url = await uploadFileToBucket({
+    file: Buffer.from((await rawAudio.arrayBuffer()) as ArrayBuffer),
+    key: `tmp/openai/audio/${createId() + createId()}.mp3`,
+    mimeType: 'audio/mpeg',
+  })
+
+  newSessionState = updateVariablesInSession(newSessionState)([
+    {
+      ...saveUrlInVariable,
+      value: url,
+    },
+  ])
+
+  return {
+    outgoingEdgeId,
+    newSessionState,
+  }
+}

packages/bot-engine/blocks/integrations/openai/executeOpenAIBlock.ts

@@ -2,6 +2,7 @@ import { SessionState } from '@typebot.io/schemas'
 import { OpenAIBlock } from '@typebot.io/schemas/features/blocks/integrations/openai'
 import { createChatCompletionOpenAI } from './createChatCompletionOpenAI'
 import { ExecuteIntegrationResponse } from '../../../types'
+import { createSpeechOpenAI } from './audio/createSpeechOpenAI'
 
 export const executeOpenAIBlock = async (
   state: SessionState,
@@ -14,6 +15,11 @@ export const executeOpenAIBlock = async (
         outgoingEdgeId: block.outgoingEdgeId,
         blockId: block.id,
       })
+    case 'Create speech':
+      return createSpeechOpenAI(state, {
+        options: block.options,
+        outgoingEdgeId: block.outgoingEdgeId,
+      })
     case 'Create image':
     case undefined:
       return { outgoingEdgeId: block.outgoingEdgeId }

packages/bot-engine/blocks/integrations/openai/parseChatCompletionMessages.ts

@@ -103,11 +103,10 @@ export const parseChatCompletionMessages =
         } satisfies OpenAI.Chat.ChatCompletionMessageParam
       })
       .filter(
-        (message) => isNotEmpty(message?.role) && isNotEmpty(message?.content)
+        (message) =>
+          isNotEmpty(message?.role) && isNotEmpty(message?.content?.toString())
       ) as OpenAI.Chat.ChatCompletionMessageParam[]
 
-    console.log('parsedMessages', parsedMessages)
-
     return {
       variablesTransformedToList,
       messages: parsedMessages,

packages/bot-engine/package.json

@@ -18,7 +18,7 @@
     "@typebot.io/tsconfig": "workspace:*",
     "@udecode/plate-common": "21.1.5",
     "@udecode/plate-serializer-md": "24.4.0",
-    "ai": "2.2.14",
+    "ai": "2.2.24",
     "chrono-node": "2.7.0",
     "date-fns": "2.30.0",
     "google-auth-library": "8.9.0",
@@ -27,7 +27,7 @@
     "libphonenumber-js": "1.10.37",
     "node-html-parser": "6.1.5",
     "nodemailer": "6.9.3",
-    "openai": "4.11.1",
+    "openai": "4.19.0",
     "qs": "6.11.2",
     "remark-slate": "1.8.6",
     "stripe": "12.13.0"