🧑‍💻 (s3) Correctly delete the files when deleting resources

packages/lib/s3/deleteFilesFromBucket.ts

@@ -1,43 +0,0 @@
-import { env } from '@typebot.io/env'
-import { Client } from 'minio'
-
-export const deleteFilesFromBucket = async ({
-  urls,
-}: {
-  urls: string[]
-}): Promise<void> => {
-  if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
-    throw new Error(
-      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
-    )
-
-  const minioClient = new Client({
-    endPoint: env.S3_ENDPOINT,
-    port: env.S3_PORT,
-    useSSL: env.S3_SSL,
-    accessKey: env.S3_ACCESS_KEY,
-    secretKey: env.S3_SECRET_KEY,
-    region: env.S3_REGION,
-  })
-
-  const bucket = env.S3_BUCKET
-
-  const keys = urls.reduce<string[]>(
-    (keys, url) => [
-      ...keys,
-      ...addKeyIfIncludesPublicCustomDomain(url),
-      ...addKeyIfIncludesDefaultEndpoint(url, bucket),
-    ],
-    []
-  )
-
-  return minioClient.removeObjects(bucket, keys)
-}
-
-const addKeyIfIncludesPublicCustomDomain = (url: string) =>
-  env.S3_PUBLIC_CUSTOM_DOMAIN && url.includes(env.S3_PUBLIC_CUSTOM_DOMAIN)
-    ? [url.split(env.S3_PUBLIC_CUSTOM_DOMAIN + '/')[1]]
-    : []
-
-const addKeyIfIncludesDefaultEndpoint = (url: string, bucket: string) =>
-  url.includes(env.S3_ENDPOINT as string) ? [url.split(`/${bucket}/`)[1]] : []

packages/lib/s3/generatePresignedPostPolicy.ts

@@ -1,5 +1,6 @@
 import { env } from '@typebot.io/env'
-import { Client, PostPolicyResult } from 'minio'
+import { PostPolicyResult } from 'minio'
+import { initClient } from './initClient'
 
 type Props = {
   filePath: string
@@ -14,19 +15,7 @@ export const generatePresignedPostPolicy = async ({
   fileType,
   maxFileSize,
 }: Props): Promise<PostPolicyResult> => {
-  if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
-    throw new Error(
-      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
-    )
-
-  const minioClient = new Client({
-    endPoint: env.S3_ENDPOINT,
-    port: env.S3_PORT,
-    useSSL: env.S3_SSL,
-    accessKey: env.S3_ACCESS_KEY,
-    secretKey: env.S3_SECRET_KEY,
-    region: env.S3_REGION,
-  })
+  const minioClient = initClient()
 
   const postPolicy = minioClient.newPostPolicy()
   if (maxFileSize)

packages/lib/s3/getFileTempUrl.ts

@@ -1,5 +1,5 @@
 import { env } from '@typebot.io/env'
-import { Client } from 'minio'
+import { initClient } from './initClient'
 
 type Props = {
   key: string
@@ -9,19 +9,6 @@ export const getFileTempUrl = async ({
   key,
   expires,
 }: Props): Promise<string> => {
-  if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
-    throw new Error(
-      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
-    )
-
-  const minioClient = new Client({
-    endPoint: env.S3_ENDPOINT,
-    port: env.S3_PORT,
-    useSSL: env.S3_SSL,
-    accessKey: env.S3_ACCESS_KEY,
-    secretKey: env.S3_SECRET_KEY,
-    region: env.S3_REGION,
-  })
-
+  const minioClient = initClient()
   return minioClient.presignedGetObject(env.S3_BUCKET, key, expires ?? 3600)
 }

packages/lib/s3/getFolderSize.ts

@@ -1,24 +1,12 @@
 import { env } from '@typebot.io/env'
-import { Client } from 'minio'
+import { initClient } from './initClient'
 
 type Props = {
   folderPath: string
 }
 
 export const getFolderSize = async ({ folderPath }: Props) => {
-  if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
-    throw new Error(
-      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
-    )
-
-  const minioClient = new Client({
-    endPoint: env.S3_ENDPOINT,
-    port: env.S3_PORT,
-    useSSL: env.S3_SSL,
-    accessKey: env.S3_ACCESS_KEY,
-    secretKey: env.S3_SECRET_KEY,
-    region: env.S3_REGION,
-  })
+  const minioClient = initClient()
 
   return new Promise<number>((resolve, reject) => {
     let totalSize = 0

packages/lib/s3/initClient.ts

@@ -0,0 +1,20 @@
+import { env } from '@typebot.io/env'
+import { Client } from 'minio'
+
+export const initClient = () => {
+  if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
+    throw new Error(
+      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
+    )
+
+  const minioClient = new Client({
+    endPoint: env.S3_ENDPOINT,
+    port: env.S3_PORT,
+    useSSL: env.S3_SSL,
+    accessKey: env.S3_ACCESS_KEY,
+    secretKey: env.S3_SECRET_KEY,
+    region: env.S3_REGION,
+  })
+
+  return minioClient
+}

packages/lib/s3/removeObjectsRecursively.ts

@@ -0,0 +1,67 @@
+import { env } from '@typebot.io/env'
+import { initClient } from './initClient'
+
+const removeObjectsRecursively = async (prefix: string) => {
+  const minioClient = initClient()
+
+  const bucketName = env.S3_BUCKET
+
+  const objectsStream = minioClient.listObjectsV2(bucketName, prefix, true)
+
+  for await (const obj of objectsStream) {
+    try {
+      await minioClient.removeObject(bucketName, obj.name)
+    } catch (err) {
+      console.error(`Error removing ${obj.name}:`, err)
+    }
+  }
+}
+
+export const removeObjectsFromWorkspace = async (workspaceId: string) => {
+  await removeObjectsRecursively(`public/workspaces/${workspaceId}/`)
+  await removeObjectsRecursively(`private/workspaces/${workspaceId}/`)
+}
+
+export const removeObjectsFromResult = async ({
+  workspaceId,
+  resultIds,
+  typebotId,
+}: {
+  workspaceId: string
+  resultIds: string[]
+  typebotId: string
+}) => {
+  for (const resultId of resultIds) {
+    await removeObjectsRecursively(
+      `public/workspaces/${workspaceId}/typebots/${typebotId}/results/${resultId}/`
+    )
+  }
+}
+
+export const removeAllObjectsFromResult = async ({
+  workspaceId,
+  typebotId,
+}: {
+  workspaceId: string
+  typebotId: string
+}) => {
+  await removeObjectsRecursively(
+    `public/workspaces/${workspaceId}/typebots/${typebotId}/results/`
+  )
+}
+
+export const removeObjectsFromTypebot = async ({
+  typebotId,
+  workspaceId,
+}: {
+  typebotId: string
+  workspaceId: string
+}) => {
+  await removeObjectsRecursively(
+    `public/workspaces/${workspaceId}/typebots/${typebotId}/`
+  )
+}
+
+export const removeObjectsFromUser = async (userId: string) => {
+  await removeObjectsRecursively(`public/users/${userId}/`)
+}

packages/lib/s3/uploadFileToBucket.ts

@@ -1,5 +1,5 @@
 import { env } from '@typebot.io/env'
-import { Client } from 'minio'
+import { initClient } from './initClient'
 
 type Props = {
   key: string
@@ -12,19 +12,7 @@ export const uploadFileToBucket = async ({
   file,
   mimeType,
 }: Props): Promise<string> => {
-  if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
-    throw new Error(
-      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
-    )
-
-  const minioClient = new Client({
-    endPoint: env.S3_ENDPOINT,
-    port: env.S3_PORT,
-    useSSL: env.S3_SSL,
-    accessKey: env.S3_ACCESS_KEY,
-    secretKey: env.S3_SECRET_KEY,
-    region: env.S3_REGION,
-  })
+  const minioClient = initClient()
 
   await minioClient.putObject(env.S3_BUCKET, 'public/' + key, file, {
     'Content-Type': mimeType,

packages/results/archiveResults.ts

@@ -1,11 +1,13 @@
 import { Prisma, PrismaClient } from '@typebot.io/prisma'
-import { Block, Typebot } from '@typebot.io/schemas'
-import { deleteFilesFromBucket } from '@typebot.io/lib/s3/deleteFilesFromBucket'
-import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants'
+import { Typebot } from '@typebot.io/schemas'
 import { isDefined } from '@typebot.io/lib'
+import {
+  removeAllObjectsFromResult,
+  removeObjectsFromResult,
+} from '@typebot.io/lib/s3/removeObjectsRecursively'
 
 type ArchiveResultsProps = {
-  typebot: Pick<Typebot, 'groups'>
+  typebot: Pick<Typebot, 'groups' | 'workspaceId' | 'id'>
   resultsFilter?: Omit<Prisma.ResultWhereInput, 'typebotId'> & {
     typebotId: string
   }
@@ -15,10 +17,6 @@ export const archiveResults =
   (prisma: PrismaClient) =>
   async ({ typebot, resultsFilter }: ArchiveResultsProps) => {
     const batchSize = 100
-    const fileUploadBlockIds = typebot.groups
-      .flatMap<Block>((group) => group.blocks)
-      .filter((block) => block.type === InputBlockType.FILE)
-      .map((block) => block.id)
 
     let currentTotalResults = 0
 
@@ -33,6 +31,8 @@ export const archiveResults =
 
     let progress = 0
 
+    const isDeletingAllResults = resultsFilter?.id === undefined
+
     do {
       progress += batchSize
       console.log(`Archiving ${progress} / ${resultsCount} results...`)
@@ -54,19 +54,6 @@ export const archiveResults =
 
       const resultIds = resultsToDelete.map((result) => result.id)
 
-      if (fileUploadBlockIds.length > 0) {
-        const filesToDelete = await prisma.answer.findMany({
-          where: {
-            resultId: { in: resultIds },
-            blockId: { in: fileUploadBlockIds },
-          },
-        })
-        if (filesToDelete.length > 0)
-          await deleteFilesFromBucket({
-            urls: filesToDelete.flatMap((a) => a.content.split(', ')),
-          })
-      }
-
       await prisma.$transaction([
         prisma.log.deleteMany({
           where: {
@@ -112,7 +99,21 @@ export const archiveResults =
           },
         }),
       ])
+      if (!isDeletingAllResults) {
+        await removeObjectsFromResult({
+          workspaceId: typebot.workspaceId,
+          resultIds: resultIds,
+          typebotId: typebot.id,
+        })
+      }
     } while (currentTotalResults >= batchSize)
 
+    if (isDeletingAllResults) {
+      await removeAllObjectsFromResult({
+        workspaceId: typebot.workspaceId,
+        typebotId: typebot.id,
+      })
+    }
+
     return { success: true }
   }

packages/scripts/destroyUser.ts

@@ -1,9 +1,3 @@
 import { destroyUser } from './helpers/destroyUser'
-import { promptAndSetEnvironment } from './utils'
 
-const runDestroyUser = async () => {
-  await promptAndSetEnvironment('production')
-  return destroyUser()
-}
-
-runDestroyUser()
+destroyUser()

packages/scripts/helpers/destroyUser.ts

@@ -1,6 +1,10 @@
 import { isCancel, text, confirm } from '@clack/prompts'
-import { Plan, PrismaClient } from '@typebot.io/prisma'
+import { PrismaClient } from '@typebot.io/prisma'
 import { writeFileSync } from 'fs'
+import {
+  removeObjectsFromUser,
+  removeObjectsFromWorkspace,
+} from '@typebot.io/lib/s3/removeObjectsRecursively'
 
 export const destroyUser = async (userEmail?: string) => {
   const prisma = new PrismaClient()
@@ -50,8 +54,16 @@ export const destroyUser = async (userEmail?: string) => {
   }
 
   console.log(
-    'Workspaces plans:',
-    workspaces.map((w) => w.plan)
+    'Workspaces:',
+    JSON.stringify(
+      workspaces.map((w) => ({
+        id: w.id,
+        plan: w.plan,
+        members: w.members,
+      })),
+      null,
+      2
+    )
   )
 
   const proceed = await confirm({ message: 'Proceed?' })
@@ -61,13 +73,15 @@ export const destroyUser = async (userEmail?: string) => {
   }
 
   for (const workspace of workspaces) {
-    const hasResults = workspace.typebots.some((t) => t.results.length > 0)
-    if (hasResults) {
+    const totalResults = workspace.typebots.reduce(
+      (acc, typebot) => acc + typebot.results.length,
+      0
+    )
+
+    if (totalResults > 0) {
       console.log(
-        `Workspace ${workspace.name} has results. Deleting results first...`,
-        workspace.typebots.filter((t) => t.results.length > 0)
+        `Workspace ${workspace.name} has ${totalResults} results. We should delete them first...`
       )
-      console.log(JSON.stringify({ members: workspace.members }, null, 2))
       const proceed = await confirm({ message: 'Proceed?' })
       if (!proceed || typeof proceed !== 'boolean') {
         console.log('Aborting')
@@ -82,9 +96,11 @@ export const destroyUser = async (userEmail?: string) => {
       }
     }
     await prisma.workspace.delete({ where: { id: workspace.id } })
+    await removeObjectsFromWorkspace(workspace.id)
   }
 
   const user = await prisma.user.delete({ where: { email } })
+  await removeObjectsFromUser(user.id)
 
-  console.log(`Deleted user ${JSON.stringify(user, null, 2)}`)
+  console.log(`User deleted.`, JSON.stringify(user, null, 2))
 }

packages/scripts/package.json

@@ -20,7 +20,7 @@
     "insertUsersInBrevoList": "tsx insertUsersInBrevoList.ts",
     "getUsage": "tsx getUsage.ts",
     "suspendWorkspace": "tsx suspendWorkspace.ts",
-    "destroyUser": "tsx destroyUser.ts",
+    "destroyUser": "SKIP_ENV_CHECK=true dotenv -e ./.env.production -- tsx destroyUser.ts",
     "updateTypebot": "tsx updateTypebot.ts",
     "updateWorkspace": "tsx updateWorkspace.ts",
     "inspectTypebot": "tsx inspectTypebot.ts",