blsmedia/bot
2
0
Fork 0
Code Activity

👷 Improve monthly clean database script

Browse Source 
Archive results that were not properly archived
This commit is contained in:
Baptiste Arnaud
2023-07-12 12:00:23 +02:00
parent c365c547aa
commit 455c3bdfd7
7 changed files with 242 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
apps/builder/src/features/results/api/deleteResults.ts
View File

@ -3,7 +3,8 @@ import { authenticatedProcedure } from '@/helpers/server/trpc'
import { TRPCError } from '@trpc/server'
import { Typebot } from '@typebot.io/schemas'
import { z } from 'zod'
import { archiveResults } from '../helpers/archiveResults'
import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
import prisma from '@/lib/prisma'
export const deleteResults = authenticatedProcedure
.meta({
@ -40,7 +41,7 @@ export const deleteResults = authenticatedProcedure
})) as Pick<Typebot, 'groups'> | null
if (!typebot)
throw new TRPCError({ code: 'NOT_FOUND', message: 'Typebot not found' })
const { success } = await archiveResults({
const { success } = await archiveResults(prisma)({
typebot,
resultsFilter: {
id: (idsArray?.length ?? 0) > 0 ? { in: idsArray } : undefined,

78
apps/builder/src/features/results/helpers/archiveResults.ts
View File

@ -1,78 +0,0 @@
import { deleteFilesFromBucket } from '@/helpers/deleteFilesFromBucket'
import prisma from '@/lib/prisma'
import { Prisma } from '@typebot.io/prisma'
import { InputBlockType, Typebot } from '@typebot.io/schemas'
const batchSize = 100
type Props = {
typebot: Pick<Typebot, 'groups'>
resultsFilter?: Omit<Prisma.ResultWhereInput, 'typebotId'> & {
typebotId: string
}
}
export const archiveResults = async ({ typebot, resultsFilter }: Props) => {
const fileUploadBlockIds = typebot.groups
.flatMap((group) => group.blocks)
.filter((block) => block.type === InputBlockType.FILE)
.map((block) => block.id)
let currentTotalResults = 0
do {
const resultsToDelete = await prisma.result.findMany({
where: {
...resultsFilter,
isArchived: false,
},
select: {
id: true,
},
take: batchSize,
})
if (resultsToDelete.length === 0) break
currentTotalResults = resultsToDelete.length
const resultIds = resultsToDelete.map((result) => result.id)
if (fileUploadBlockIds.length > 0) {
const filesToDelete = await prisma.answer.findMany({
where: {
resultId: { in: resultIds },
blockId: { in: fileUploadBlockIds },
},
})
if (filesToDelete.length > 0)
await deleteFilesFromBucket({
urls: filesToDelete.flatMap((a) => a.content.split(', ')),
})
}
await prisma.$transaction([
prisma.log.deleteMany({
where: {
resultId: { in: resultIds },
},
}),
prisma.answer.deleteMany({
where: {
resultId: { in: resultIds },
},
}),
prisma.result.updateMany({
where: {
id: { in: resultIds },
},
data: {
isArchived: true,
variables: [],
},
}),
])
} while (currentTotalResults >= batchSize)
return { success: true }
}

4
apps/builder/src/pages/api/typebots/[typebotId].ts
View File

@ -6,10 +6,10 @@ import { getAuthenticatedUser } from '@/features/auth/helpers/getAuthenticatedUs
import { Typebot } from '@typebot.io/schemas'
import { omit } from '@typebot.io/lib'
import { getTypebot } from '@/features/typebot/helpers/getTypebot'
import { archiveResults } from '@/features/results/helpers/archiveResults'
import { isReadTypebotForbidden } from '@/features/typebot/helpers/isReadTypebotForbidden'
import { removeTypebotOldProperties } from '@/features/typebot/helpers/removeTypebotOldProperties'
import { roundGroupsCoordinate } from '@/features/typebot/helpers/roundGroupsCoordinate'
import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
const handler = async (req: NextApiRequest, res: NextApiResponse) => {
const user = await getAuthenticatedUser(req, res)
@ -56,7 +56,7 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => {
},
})) as Pick<Typebot, 'groups'> | null
if (!typebot) return res.status(404).send({ typebot: null })
const { success } = await archiveResults({
const { success } = await archiveResults(prisma)({
typebot,
resultsFilter: { typebotId },
})

126
packages/lib/api/helpers/archiveResults.ts Normal file
View File

@ -0,0 +1,126 @@
import { Prisma, PrismaClient } from '@typebot.io/prisma'
import { InputBlockType, Typebot } from '@typebot.io/schemas'
import { Client } from 'minio'
type ArchiveResultsProps = {
typebot: Pick<Typebot, 'groups'>
resultsFilter?: Omit<Prisma.ResultWhereInput, 'typebotId'> & {
typebotId: string
}
}
export const archiveResults =
(prisma: PrismaClient) =>
async ({ typebot, resultsFilter }: ArchiveResultsProps) => {
const batchSize = 100
const fileUploadBlockIds = typebot.groups
.flatMap((group) => group.blocks)
.filter((block) => block.type === InputBlockType.FILE)
.map((block) => block.id)
let currentTotalResults = 0
const resultsCount = await prisma.result.count({
where: {
...resultsFilter,
OR: [{ isArchived: false }, { isArchived: null }],
},
})
if (resultsCount === 0) return { success: true }
let progress = 0
do {
progress += batchSize
console.log(`Archiving ${progress} / ${resultsCount} results...`)
const resultsToDelete = await prisma.result.findMany({
where: {
...resultsFilter,
OR: [{ isArchived: false }, { isArchived: null }],
},
select: {
id: true,
},
take: batchSize,
})
if (resultsToDelete.length === 0) break
currentTotalResults = resultsToDelete.length
const resultIds = resultsToDelete.map((result) => result.id)
if (fileUploadBlockIds.length > 0) {
const filesToDelete = await prisma.answer.findMany({
where: {
resultId: { in: resultIds },
blockId: { in: fileUploadBlockIds },
},
})
if (filesToDelete.length > 0)
await deleteFilesFromBucket({
urls: filesToDelete.flatMap((a) => a.content.split(', ')),
})
}
await prisma.$transaction([
prisma.log.deleteMany({
where: {
resultId: { in: resultIds },
},
}),
prisma.answer.deleteMany({
where: {
resultId: { in: resultIds },
},
}),
prisma.result.updateMany({
where: {
id: { in: resultIds },
},
data: {
isArchived: true,
variables: [],
},
}),
])
} while (currentTotalResults >= batchSize)
return { success: true }
}
const deleteFilesFromBucket = async ({
urls,
}: {
urls: string[]
}): Promise<void> => {
if (
!process.env.S3_ENDPOINT ||
!process.env.S3_ACCESS_KEY ||
!process.env.S3_SECRET_KEY
)
throw new Error(
'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
)
const useSSL =
process.env.S3_SSL && process.env.S3_SSL === 'false' ? false : true
const minioClient = new Client({
endPoint: process.env.S3_ENDPOINT,
port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : undefined,
useSSL,
accessKey: process.env.S3_ACCESS_KEY,
secretKey: process.env.S3_SECRET_KEY,
region: process.env.S3_REGION,
})
const bucket = process.env.S3_BUCKET ?? 'typebot'
return minioClient.removeObjects(
bucket,
urls
.filter((url) => url.includes(process.env.S3_ENDPOINT as string))
.map((url) => url.split(`/${bucket}/`)[1])
)
}

9
packages/lib/package.json
View File

@ -8,14 +8,14 @@
"devDependencies": {
"@paralleldrive/cuid2": "2.2.0",
"@playwright/test": "1.34.3",
"@typebot.io/prisma": "workspace:*",
"@typebot.io/schemas": "workspace:*",
"@typebot.io/tsconfig": "workspace:*",
"@types/nodemailer": "6.4.8",
"aws-sdk": "2.1384.0",
"@typebot.io/prisma": "workspace:*",
"dotenv": "16.0.3",
"@typebot.io/schemas": "workspace:*",
"next": "13.4.3",
"nodemailer": "6.9.2",
"@typebot.io/tsconfig": "workspace:*",
"typescript": "5.0.4"
},
"peerDependencies": {
@ -24,6 +24,7 @@
"nodemailer": "6.7.8"
},
"dependencies": {
"got": "12.6.0"
"got": "12.6.0",
"minio": "7.1.1"
}
}

91
packages/scripts/cleanDatabase.ts
View File

@ -1,5 +1,7 @@
import { PrismaClient } from '@typebot.io/prisma'
import { promptAndSetEnvironment } from './utils'
import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
import { Typebot } from '@typebot.io/schemas'
const prisma = new PrismaClient()
@ -14,9 +16,9 @@ export const cleanDatabase = async () => {
if (isFirstOfMonth) {
await deleteArchivedResults()
await deleteArchivedTypebots()
await resetQuarantinedWorkspaces()
await resetBillingProps()
}
console.log('Done!')
console.log('Database cleaned!')
}
const deleteArchivedTypebots = async () => {
@ -24,23 +26,39 @@ const deleteArchivedTypebots = async () => {
lastDayTwoMonthsAgo.setMonth(lastDayTwoMonthsAgo.getMonth() - 1)
lastDayTwoMonthsAgo.setDate(0)
const { count } = await prisma.typebot.deleteMany({
const typebots = await prisma.typebot.findMany({
where: {
updatedAt: {
lte: lastDayTwoMonthsAgo,
},
isArchived: true,
},
select: { id: true },
})
console.log(`Deleted ${count} archived typebots.`)
console.log(`Deleting ${typebots.length} archived typebots...`)
const chunkSize = 1000
for (let i = 0; i < typebots.length; i += chunkSize) {
const chunk = typebots.slice(i, i + chunkSize)
await deleteResultsFromArchivedTypebotsIfAny(chunk)
await prisma.typebot.deleteMany({
where: {
id: {
in: chunk.map((typebot) => typebot.id),
},
},
})
}
console.log('Done!')
}
const deleteArchivedResults = async () => {
const lastDayTwoMonthsAgo = new Date()
lastDayTwoMonthsAgo.setMonth(lastDayTwoMonthsAgo.getMonth() - 1)
lastDayTwoMonthsAgo.setDate(0)
let totalResults
do {
const results = await prisma.result.findMany({
where: {
createdAt: {
@ -49,8 +67,9 @@ const deleteArchivedResults = async () => {
isArchived: true,
},
select: { id: true },
take: 80000,
})
totalResults = results.length
console.log(`Deleting ${results.length} archived results...`)
const chunkSize = 1000
for (let i = 0; i < results.length; i += chunkSize) {
@ -63,6 +82,9 @@ const deleteArchivedResults = async () => {
},
})
}
} while (totalResults === 80000)
console.log('Done!')
}
const deleteOldChatSessions = async () => {
@ -144,16 +166,69 @@ const deleteExpiredVerificationTokens = async () => {
})
}
} while (totalVerificationTokens === 80000)
console.log('Done!')
}
const resetQuarantinedWorkspaces = async () =>
prisma.workspace.updateMany({
const resetBillingProps = async () => {
console.log('Resetting billing props...')
const { count } = await prisma.workspace.updateMany({
where: {
OR: [
{
isQuarantined: true,
},
{
chatsLimitFirstEmailSentAt: { not: null },
},
{
storageLimitFirstEmailSentAt: { not: null },
},
],
},
data: {
isQuarantined: false,
chatsLimitFirstEmailSentAt: null,
storageLimitFirstEmailSentAt: null,
chatsLimitSecondEmailSentAt: null,
storageLimitSecondEmailSentAt: null,
},
})
console.log(`Resetted ${count} workspaces.`)
}
const deleteResultsFromArchivedTypebotsIfAny = async (
typebotIds: { id: string }[]
) => {
console.log('Checking for archived typebots with non-archived results...')
const archivedTypebotsWithResults = (await prisma.typebot.findMany({
where: {
id: {
in: typebotIds.map((typebot) => typebot.id),
},
isArchived: true,
results: {
some: {},
},
},
select: {
id: true,
groups: true,
},
})) as Pick<Typebot, 'groups' | 'id'>[]
if (archivedTypebotsWithResults.length === 0) return
console.log(
`Found ${archivedTypebotsWithResults.length} archived typebots with non-archived results.`
)
for (const archivedTypebot of archivedTypebotsWithResults) {
await archiveResults(prisma)({
typebot: archivedTypebot,
resultsFilter: {
typebotId: archivedTypebot.id,
},
})
}
console.log('Delete archived results...')
await deleteArchivedResults()
}
cleanDatabase().then()

3
pnpm-lock.yaml generated
View File

@ -1002,6 +1002,9 @@ importers:
got:
specifier: 12.6.0
version: 12.6.0
minio:
specifier: 7.1.1
version: 7.1.1
devDependencies:
'@paralleldrive/cuid2':
specifier: 2.2.0