👷 Improve monthly clean database script

Archive results that were not properly archived

apps/builder/src/features/results/api/deleteResults.ts

@@ -3,7 +3,8 @@ import { authenticatedProcedure } from '@/helpers/server/trpc'
 import { TRPCError } from '@trpc/server'
 import { Typebot } from '@typebot.io/schemas'
 import { z } from 'zod'
-import { archiveResults } from '../helpers/archiveResults'
+import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
+import prisma from '@/lib/prisma'
 
 export const deleteResults = authenticatedProcedure
   .meta({
@@ -40,7 +41,7 @@ export const deleteResults = authenticatedProcedure
     })) as Pick<Typebot, 'groups'> | null
     if (!typebot)
       throw new TRPCError({ code: 'NOT_FOUND', message: 'Typebot not found' })
-    const { success } = await archiveResults({
+    const { success } = await archiveResults(prisma)({
       typebot,
       resultsFilter: {
         id: (idsArray?.length ?? 0) > 0 ? { in: idsArray } : undefined,

apps/builder/src/features/results/helpers/archiveResults.ts

@@ -1,78 +0,0 @@
-import { deleteFilesFromBucket } from '@/helpers/deleteFilesFromBucket'
-import prisma from '@/lib/prisma'
-import { Prisma } from '@typebot.io/prisma'
-import { InputBlockType, Typebot } from '@typebot.io/schemas'
-
-const batchSize = 100
-
-type Props = {
-  typebot: Pick<Typebot, 'groups'>
-  resultsFilter?: Omit<Prisma.ResultWhereInput, 'typebotId'> & {
-    typebotId: string
-  }
-}
-
-export const archiveResults = async ({ typebot, resultsFilter }: Props) => {
-  const fileUploadBlockIds = typebot.groups
-    .flatMap((group) => group.blocks)
-    .filter((block) => block.type === InputBlockType.FILE)
-    .map((block) => block.id)
-
-  let currentTotalResults = 0
-
-  do {
-    const resultsToDelete = await prisma.result.findMany({
-      where: {
-        ...resultsFilter,
-        isArchived: false,
-      },
-      select: {
-        id: true,
-      },
-      take: batchSize,
-    })
-
-    if (resultsToDelete.length === 0) break
-
-    currentTotalResults = resultsToDelete.length
-
-    const resultIds = resultsToDelete.map((result) => result.id)
-
-    if (fileUploadBlockIds.length > 0) {
-      const filesToDelete = await prisma.answer.findMany({
-        where: {
-          resultId: { in: resultIds },
-          blockId: { in: fileUploadBlockIds },
-        },
-      })
-      if (filesToDelete.length > 0)
-        await deleteFilesFromBucket({
-          urls: filesToDelete.flatMap((a) => a.content.split(', ')),
-        })
-    }
-
-    await prisma.$transaction([
-      prisma.log.deleteMany({
-        where: {
-          resultId: { in: resultIds },
-        },
-      }),
-      prisma.answer.deleteMany({
-        where: {
-          resultId: { in: resultIds },
-        },
-      }),
-      prisma.result.updateMany({
-        where: {
-          id: { in: resultIds },
-        },
-        data: {
-          isArchived: true,
-          variables: [],
-        },
-      }),
-    ])
-  } while (currentTotalResults >= batchSize)
-
-  return { success: true }
-}

apps/builder/src/pages/api/typebots/[typebotId].ts

@@ -6,10 +6,10 @@ import { getAuthenticatedUser } from '@/features/auth/helpers/getAuthenticatedUs
 import { Typebot } from '@typebot.io/schemas'
 import { omit } from '@typebot.io/lib'
 import { getTypebot } from '@/features/typebot/helpers/getTypebot'
-import { archiveResults } from '@/features/results/helpers/archiveResults'
 import { isReadTypebotForbidden } from '@/features/typebot/helpers/isReadTypebotForbidden'
 import { removeTypebotOldProperties } from '@/features/typebot/helpers/removeTypebotOldProperties'
 import { roundGroupsCoordinate } from '@/features/typebot/helpers/roundGroupsCoordinate'
+import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
 
 const handler = async (req: NextApiRequest, res: NextApiResponse) => {
   const user = await getAuthenticatedUser(req, res)
@@ -56,7 +56,7 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => {
       },
     })) as Pick<Typebot, 'groups'> | null
     if (!typebot) return res.status(404).send({ typebot: null })
-    const { success } = await archiveResults({
+    const { success } = await archiveResults(prisma)({
       typebot,
       resultsFilter: { typebotId },
     })

packages/lib/api/helpers/archiveResults.ts

@@ -0,0 +1,126 @@
+import { Prisma, PrismaClient } from '@typebot.io/prisma'
+import { InputBlockType, Typebot } from '@typebot.io/schemas'
+import { Client } from 'minio'
+
+type ArchiveResultsProps = {
+  typebot: Pick<Typebot, 'groups'>
+  resultsFilter?: Omit<Prisma.ResultWhereInput, 'typebotId'> & {
+    typebotId: string
+  }
+}
+
+export const archiveResults =
+  (prisma: PrismaClient) =>
+  async ({ typebot, resultsFilter }: ArchiveResultsProps) => {
+    const batchSize = 100
+    const fileUploadBlockIds = typebot.groups
+      .flatMap((group) => group.blocks)
+      .filter((block) => block.type === InputBlockType.FILE)
+      .map((block) => block.id)
+
+    let currentTotalResults = 0
+
+    const resultsCount = await prisma.result.count({
+      where: {
+        ...resultsFilter,
+        OR: [{ isArchived: false }, { isArchived: null }],
+      },
+    })
+
+    if (resultsCount === 0) return { success: true }
+
+    let progress = 0
+
+    do {
+      progress += batchSize
+      console.log(`Archiving ${progress} / ${resultsCount} results...`)
+      const resultsToDelete = await prisma.result.findMany({
+        where: {
+          ...resultsFilter,
+          OR: [{ isArchived: false }, { isArchived: null }],
+        },
+        select: {
+          id: true,
+        },
+        take: batchSize,
+      })
+
+      if (resultsToDelete.length === 0) break
+
+      currentTotalResults = resultsToDelete.length
+
+      const resultIds = resultsToDelete.map((result) => result.id)
+
+      if (fileUploadBlockIds.length > 0) {
+        const filesToDelete = await prisma.answer.findMany({
+          where: {
+            resultId: { in: resultIds },
+            blockId: { in: fileUploadBlockIds },
+          },
+        })
+        if (filesToDelete.length > 0)
+          await deleteFilesFromBucket({
+            urls: filesToDelete.flatMap((a) => a.content.split(', ')),
+          })
+      }
+
+      await prisma.$transaction([
+        prisma.log.deleteMany({
+          where: {
+            resultId: { in: resultIds },
+          },
+        }),
+        prisma.answer.deleteMany({
+          where: {
+            resultId: { in: resultIds },
+          },
+        }),
+        prisma.result.updateMany({
+          where: {
+            id: { in: resultIds },
+          },
+          data: {
+            isArchived: true,
+            variables: [],
+          },
+        }),
+      ])
+    } while (currentTotalResults >= batchSize)
+
+    return { success: true }
+  }
+
+const deleteFilesFromBucket = async ({
+  urls,
+}: {
+  urls: string[]
+}): Promise<void> => {
+  if (
+    !process.env.S3_ENDPOINT ||
+    !process.env.S3_ACCESS_KEY ||
+    !process.env.S3_SECRET_KEY
+  )
+    throw new Error(
+      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
+    )
+
+  const useSSL =
+    process.env.S3_SSL && process.env.S3_SSL === 'false' ? false : true
+  const minioClient = new Client({
+    endPoint: process.env.S3_ENDPOINT,
+    port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : undefined,
+    useSSL,
+    accessKey: process.env.S3_ACCESS_KEY,
+    secretKey: process.env.S3_SECRET_KEY,
+    region: process.env.S3_REGION,
+  })
+
+  const bucket = process.env.S3_BUCKET ?? 'typebot'
+
+  return minioClient.removeObjects(
+    bucket,
+    urls
+      .filter((url) => url.includes(process.env.S3_ENDPOINT as string))
+      .map((url) => url.split(`/${bucket}/`)[1])
+  )
+}

packages/lib/package.json

@@ -8,14 +8,14 @@
   "devDependencies": {
     "@paralleldrive/cuid2": "2.2.0",
     "@playwright/test": "1.34.3",
+    "@typebot.io/prisma": "workspace:*",
+    "@typebot.io/schemas": "workspace:*",
+    "@typebot.io/tsconfig": "workspace:*",
     "@types/nodemailer": "6.4.8",
     "aws-sdk": "2.1384.0",
-    "@typebot.io/prisma": "workspace:*",
     "dotenv": "16.0.3",
-    "@typebot.io/schemas": "workspace:*",
     "next": "13.4.3",
     "nodemailer": "6.9.2",
-    "@typebot.io/tsconfig": "workspace:*",
     "typescript": "5.0.4"
   },
   "peerDependencies": {
@@ -24,6 +24,7 @@
     "nodemailer": "6.7.8"
   },
   "dependencies": {
-    "got": "12.6.0"
+    "got": "12.6.0",
+    "minio": "7.1.1"
   }
 }

packages/scripts/cleanDatabase.ts

@@ -1,5 +1,7 @@
 import { PrismaClient } from '@typebot.io/prisma'
 import { promptAndSetEnvironment } from './utils'
+import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
+import { Typebot } from '@typebot.io/schemas'
 
 const prisma = new PrismaClient()
 
@@ -14,9 +16,9 @@ export const cleanDatabase = async () => {
   if (isFirstOfMonth) {
     await deleteArchivedResults()
     await deleteArchivedTypebots()
-    await resetQuarantinedWorkspaces()
+    await resetBillingProps()
   }
-  console.log('Done!')
+  console.log('Database cleaned!')
 }
 
 const deleteArchivedTypebots = async () => {
@@ -24,45 +26,65 @@ const deleteArchivedTypebots = async () => {
   lastDayTwoMonthsAgo.setMonth(lastDayTwoMonthsAgo.getMonth() - 1)
   lastDayTwoMonthsAgo.setDate(0)
 
-  const { count } = await prisma.typebot.deleteMany({
+  const typebots = await prisma.typebot.findMany({
     where: {
       updatedAt: {
         lte: lastDayTwoMonthsAgo,
       },
       isArchived: true,
     },
-  })
-
-  console.log(`Deleted ${count} archived typebots.`)
-}
-
-const deleteArchivedResults = async () => {
-  const lastDayTwoMonthsAgo = new Date()
-  lastDayTwoMonthsAgo.setMonth(lastDayTwoMonthsAgo.getMonth() - 1)
-  lastDayTwoMonthsAgo.setDate(0)
-
-  const results = await prisma.result.findMany({
-    where: {
-      createdAt: {
-        lte: lastDayTwoMonthsAgo,
-      },
-      isArchived: true,
-    },
     select: { id: true },
   })
 
-  console.log(`Deleting ${results.length} archived results...`)
+  console.log(`Deleting ${typebots.length} archived typebots...`)
+
   const chunkSize = 1000
-  for (let i = 0; i < results.length; i += chunkSize) {
+  for (let i = 0; i < typebots.length; i += chunkSize) {
-    const chunk = results.slice(i, i + chunkSize)
+    const chunk = typebots.slice(i, i + chunkSize)
-    await prisma.result.deleteMany({
+    await deleteResultsFromArchivedTypebotsIfAny(chunk)
+    await prisma.typebot.deleteMany({
       where: {
         id: {
-          in: chunk.map((result) => result.id),
+          in: chunk.map((typebot) => typebot.id),
         },
       },
     })
   }
+  console.log('Done!')
+}
+
+const deleteArchivedResults = async () => {
+  const lastDayTwoMonthsAgo = new Date()
+  lastDayTwoMonthsAgo.setMonth(lastDayTwoMonthsAgo.getMonth() - 1)
+  lastDayTwoMonthsAgo.setDate(0)
+  let totalResults
+  do {
+    const results = await prisma.result.findMany({
+      where: {
+        createdAt: {
+          lte: lastDayTwoMonthsAgo,
+        },
+        isArchived: true,
+      },
+      select: { id: true },
+      take: 80000,
+    })
+    totalResults = results.length
+    console.log(`Deleting ${results.length} archived results...`)
+    const chunkSize = 1000
+    for (let i = 0; i < results.length; i += chunkSize) {
+      const chunk = results.slice(i, i + chunkSize)
+      await prisma.result.deleteMany({
+        where: {
+          id: {
+            in: chunk.map((result) => result.id),
+          },
+        },
+      })
+    }
+  } while (totalResults === 80000)
+
+  console.log('Done!')
 }
 
 const deleteOldChatSessions = async () => {
@@ -144,16 +166,69 @@ const deleteExpiredVerificationTokens = async () => {
       })
     }
   } while (totalVerificationTokens === 80000)
+  console.log('Done!')
 }
 
-const resetQuarantinedWorkspaces = async () =>
+const resetBillingProps = async () => {
-  prisma.workspace.updateMany({
+  console.log('Resetting billing props...')
+  const { count } = await prisma.workspace.updateMany({
     where: {
-      isQuarantined: true,
+      OR: [
+        {
+          isQuarantined: true,
+        },
+        {
+          chatsLimitFirstEmailSentAt: { not: null },
+        },
+        {
+          storageLimitFirstEmailSentAt: { not: null },
+        },
+      ],
     },
     data: {
       isQuarantined: false,
+      chatsLimitFirstEmailSentAt: null,
+      storageLimitFirstEmailSentAt: null,
+      chatsLimitSecondEmailSentAt: null,
+      storageLimitSecondEmailSentAt: null,
     },
   })
+  console.log(`Resetted ${count} workspaces.`)
+}
+
+const deleteResultsFromArchivedTypebotsIfAny = async (
+  typebotIds: { id: string }[]
+) => {
+  console.log('Checking for archived typebots with non-archived results...')
+  const archivedTypebotsWithResults = (await prisma.typebot.findMany({
+    where: {
+      id: {
+        in: typebotIds.map((typebot) => typebot.id),
+      },
+      isArchived: true,
+      results: {
+        some: {},
+      },
+    },
+    select: {
+      id: true,
+      groups: true,
+    },
+  })) as Pick<Typebot, 'groups' | 'id'>[]
+  if (archivedTypebotsWithResults.length === 0) return
+  console.log(
+    `Found ${archivedTypebotsWithResults.length} archived typebots with non-archived results.`
+  )
+  for (const archivedTypebot of archivedTypebotsWithResults) {
+    await archiveResults(prisma)({
+      typebot: archivedTypebot,
+      resultsFilter: {
+        typebotId: archivedTypebot.id,
+      },
+    })
+  }
+  console.log('Delete archived results...')
+  await deleteArchivedResults()
+}
 
 cleanDatabase().then()

pnpm-lock.yaml

@@ -1002,6 +1002,9 @@ importers:
       got:
         specifier: 12.6.0
         version: 12.6.0
+      minio:
+        specifier: 7.1.1
+        version: 7.1.1
     devDependencies:
       '@paralleldrive/cuid2':
         specifier: 2.2.0