✨ (openai) Add create speech OpenAI action

Closes #1025

packages/bot-engine/blocks/integrations/openai/audio/createSpeechOpenAI.ts

@@ -0,0 +1,121 @@
+import { SessionState } from '@typebot.io/schemas'
+import {
+  CreateSpeechOpenAIOptions,
+  OpenAICredentials,
+} from '@typebot.io/schemas/features/blocks/integrations/openai'
+import { isNotEmpty } from '@typebot.io/lib'
+import { decrypt } from '@typebot.io/lib/api/encryption/decrypt'
+import prisma from '@typebot.io/lib/prisma'
+import { defaultOpenAIOptions } from '@typebot.io/schemas/features/blocks/integrations/openai/constants'
+import { ExecuteIntegrationResponse } from '../../../../types'
+import OpenAI, { ClientOptions } from 'openai'
+import { uploadFileToBucket } from '@typebot.io/lib/s3/uploadFileToBucket'
+import { updateVariablesInSession } from '../../../../variables/updateVariablesInSession'
+import { createId } from '@paralleldrive/cuid2'
+import { parseVariables } from '../../../../variables/parseVariables'
+
+export const createSpeechOpenAI = async (
+  state: SessionState,
+  {
+    outgoingEdgeId,
+    options,
+  }: {
+    outgoingEdgeId?: string
+    options: CreateSpeechOpenAIOptions
+  }
+): Promise<ExecuteIntegrationResponse> => {
+  let newSessionState = state
+  const noCredentialsError = {
+    status: 'error',
+    description: 'Make sure to select an OpenAI account',
+  }
+
+  if (!options.input || !options.voice || !options.saveUrlInVariableId) {
+    return {
+      outgoingEdgeId,
+      logs: [
+        {
+          status: 'error',
+          description:
+            'Make sure to enter an input, select a voice and select a variable to save the URL in',
+        },
+      ],
+    }
+  }
+
+  if (!options.credentialsId) {
+    return {
+      outgoingEdgeId,
+      logs: [noCredentialsError],
+    }
+  }
+  const credentials = await prisma.credentials.findUnique({
+    where: {
+      id: options.credentialsId,
+    },
+  })
+  if (!credentials) {
+    console.error('Could not find credentials in database')
+    return { outgoingEdgeId, logs: [noCredentialsError] }
+  }
+  const { apiKey } = (await decrypt(
+    credentials.data,
+    credentials.iv
+  )) as OpenAICredentials['data']
+
+  const config = {
+    apiKey,
+    baseURL: options.baseUrl ?? defaultOpenAIOptions.baseUrl,
+    defaultHeaders: {
+      'api-key': apiKey,
+    },
+    defaultQuery: isNotEmpty(options.apiVersion)
+      ? {
+          'api-version': options.apiVersion,
+        }
+      : undefined,
+  } satisfies ClientOptions
+
+  const openai = new OpenAI(config)
+
+  const variables = newSessionState.typebotsQueue[0].typebot.variables
+  const saveUrlInVariable = variables.find(
+    (v) => v.id === options.saveUrlInVariableId
+  )
+
+  if (!saveUrlInVariable) {
+    return {
+      outgoingEdgeId,
+      logs: [
+        {
+          status: 'error',
+          description: 'Could not find variable to save URL in',
+        },
+      ],
+    }
+  }
+
+  const rawAudio = (await openai.audio.speech.create({
+    input: parseVariables(variables)(options.input),
+    voice: options.voice,
+    model: options.model as 'tts-1' | 'tts-1-hd',
+  })) as any
+
+  const url = await uploadFileToBucket({
+    file: Buffer.from((await rawAudio.arrayBuffer()) as ArrayBuffer),
+    key: `tmp/openai/audio/${createId() + createId()}.mp3`,
+    mimeType: 'audio/mpeg',
+  })
+
+  newSessionState = updateVariablesInSession(newSessionState)([
+    {
+      ...saveUrlInVariable,
+      value: url,
+    },
+  ])
+
+  return {
+    outgoingEdgeId,
+    newSessionState,
+  }
+}

packages/bot-engine/blocks/integrations/openai/executeOpenAIBlock.ts

@@ -2,6 +2,7 @@ import { SessionState } from '@typebot.io/schemas'
 import { OpenAIBlock } from '@typebot.io/schemas/features/blocks/integrations/openai'
 import { createChatCompletionOpenAI } from './createChatCompletionOpenAI'
 import { ExecuteIntegrationResponse } from '../../../types'
+import { createSpeechOpenAI } from './audio/createSpeechOpenAI'
 
 export const executeOpenAIBlock = async (
   state: SessionState,
@@ -14,6 +15,11 @@ export const executeOpenAIBlock = async (
         outgoingEdgeId: block.outgoingEdgeId,
         blockId: block.id,
       })
+    case 'Create speech':
+      return createSpeechOpenAI(state, {
+        options: block.options,
+        outgoingEdgeId: block.outgoingEdgeId,
+      })
     case 'Create image':
     case undefined:
       return { outgoingEdgeId: block.outgoingEdgeId }

packages/bot-engine/blocks/integrations/openai/parseChatCompletionMessages.ts

@@ -103,11 +103,10 @@ export const parseChatCompletionMessages =
         } satisfies OpenAI.Chat.ChatCompletionMessageParam
       })
       .filter(
-        (message) => isNotEmpty(message?.role) && isNotEmpty(message?.content)
+        (message) =>
+          isNotEmpty(message?.role) && isNotEmpty(message?.content?.toString())
       ) as OpenAI.Chat.ChatCompletionMessageParam[]
 
-    console.log('parsedMessages', parsedMessages)
-
     return {
       variablesTransformedToList,
       messages: parsedMessages,

packages/bot-engine/package.json

@@ -18,7 +18,7 @@
     "@typebot.io/tsconfig": "workspace:*",
     "@udecode/plate-common": "21.1.5",
     "@udecode/plate-serializer-md": "24.4.0",
-    "ai": "2.2.14",
+    "ai": "2.2.24",
     "chrono-node": "2.7.0",
     "date-fns": "2.30.0",
     "google-auth-library": "8.9.0",
@@ -27,7 +27,7 @@
     "libphonenumber-js": "1.10.37",
     "node-html-parser": "6.1.5",
     "nodemailer": "6.9.3",
-    "openai": "4.11.1",
+    "openai": "4.19.0",
     "qs": "6.11.2",
     "remark-slate": "1.8.6",
     "stripe": "12.13.0"

packages/lib/s3/uploadFileToBucket.ts

@@ -2,13 +2,13 @@ import { env } from '@typebot.io/env'
 import { Client } from 'minio'
 
 type Props = {
-  fileName: string
+  key: string
   file: Buffer
   mimeType: string
 }
 
 export const uploadFileToBucket = async ({
-  fileName,
+  key,
   file,
   mimeType,
 }: Props): Promise<string> => {
@@ -26,11 +26,14 @@ export const uploadFileToBucket = async ({
     region: env.S3_REGION,
   })
 
-  await minioClient.putObject(env.S3_BUCKET, fileName, file, {
+  await minioClient.putObject(env.S3_BUCKET, 'public/' + key, file, {
     'Content-Type': mimeType,
+    'Cache-Control': 'public, max-age=86400',
   })
 
-  return `http${env.S3_SSL ? 's' : ''}://${env.S3_ENDPOINT}${
-    env.S3_PORT ? `:${env.S3_PORT}` : ''
-  }/${env.S3_BUCKET}/${fileName}`
+  return env.S3_PUBLIC_CUSTOM_DOMAIN
+    ? `${env.S3_PUBLIC_CUSTOM_DOMAIN}/public/${key}`
+    : `http${env.S3_SSL ? 's' : ''}://${env.S3_ENDPOINT}${
+        env.S3_PORT ? `:${env.S3_PORT}` : ''
+      }/${env.S3_BUCKET}/public/${key}`
 }

packages/schemas/features/blocks/integrations/openai/constants.ts

@@ -1,4 +1,8 @@
-export const openAITasks = ['Create chat completion', 'Create image'] as const
+export const openAITasks = [
+  'Create chat completion',
+  'Create speech',
+  'Create image',
+] as const
 
 export const chatCompletionMessageRoles = [
   'system',
@@ -27,3 +31,12 @@ export const defaultOpenAIOptions = {
 export const defaultOpenAIResponseMappingItem = {
   valueToExtract: 'Message content',
 } as const
+
+export const openAIVoices = [
+  'alloy',
+  'echo',
+  'fable',
+  'onyx',
+  'nova',
+  'shimmer',
+] as const

packages/schemas/features/blocks/integrations/openai/schema.ts

@@ -4,6 +4,7 @@ import {
   chatCompletionMessageRoles,
   chatCompletionResponseValues,
   openAITasks,
+  openAIVoices,
 } from './constants'
 import { variableStringSchema } from '../../../utils'
 import { blockBaseSchema, credentialsBaseSchema } from '../../shared'
@@ -78,10 +79,13 @@ const chatCompletionOptionsSchema = z
       .optional(),
   })
   .merge(openAIBaseOptionsSchema)
+export type ChatCompletionOpenAIOptions = z.infer<
+  typeof chatCompletionOptionsSchema
+>
 
 const createImageOptionsSchema = z
   .object({
-    task: z.literal(openAITasks[1]),
+    task: z.literal(openAITasks[2]),
     prompt: z.string().optional(),
     advancedOptions: z.object({
       size: z.enum(['256x256', '512x512', '1024x1024']).optional(),
@@ -95,6 +99,18 @@ const createImageOptionsSchema = z
     ),
   })
   .merge(openAIBaseOptionsSchema)
+export type CreateImageOpenAIOptions = z.infer<typeof createImageOptionsSchema>
+
+const createSpeechOptionsSchema = openAIBaseOptionsSchema.extend({
+  task: z.literal(openAITasks[1]),
+  model: z.string().optional(),
+  input: z.string().optional(),
+  voice: z.enum(openAIVoices).optional(),
+  saveUrlInVariableId: z.string().optional(),
+})
+export type CreateSpeechOpenAIOptions = z.infer<
+  typeof createSpeechOptionsSchema
+>
 
 export const openAIBlockSchema = blockBaseSchema.merge(
   z.object({
@@ -104,10 +120,12 @@ export const openAIBlockSchema = blockBaseSchema.merge(
         initialOptionsSchema,
         chatCompletionOptionsSchema,
         createImageOptionsSchema,
+        createSpeechOptionsSchema,
       ])
       .optional(),
   })
 )
+export type OpenAIBlock = z.infer<typeof openAIBlockSchema>
 
 export const openAICredentialsSchema = z
   .object({
@@ -117,10 +135,4 @@ export const openAICredentialsSchema = z
     }),
   })
   .merge(credentialsBaseSchema)
-
 export type OpenAICredentials = z.infer<typeof openAICredentialsSchema>
-export type OpenAIBlock = z.infer<typeof openAIBlockSchema>
-export type ChatCompletionOpenAIOptions = z.infer<
-  typeof chatCompletionOptionsSchema
->
-export type CreateImageOpenAIOptions = z.infer<typeof createImageOptionsSchema>