2
0

🧑‍💻 (s3) Correctly delete the files when deleting resources

This commit is contained in:
Baptiste Arnaud
2024-09-02 11:23:01 +02:00
parent f53705268c
commit 041b817aa0
16 changed files with 225 additions and 206 deletions

View File

@ -43,6 +43,7 @@ export const deleteResults = authenticatedProcedure
groups: true,
workspace: {
select: {
id: true,
isSuspended: true,
isPastDue: true,
members: {
@ -65,8 +66,10 @@ export const deleteResults = authenticatedProcedure
throw new TRPCError({ code: 'NOT_FOUND', message: 'Typebot not found' })
const { success } = await archiveResults(prisma)({
typebot: {
groups: typebot.groups,
} as Pick<Typebot, 'groups'>,
id: typebotId,
workspaceId: typebot.workspace.id,
groups: typebot.groups as Typebot['groups'],
},
resultsFilter: {
id: (idsArray?.length ?? 0) > 0 ? { in: idsArray } : undefined,
typebotId,

View File

@ -5,6 +5,7 @@ import { Typebot } from '@typebot.io/schemas'
import { z } from 'zod'
import { isWriteTypebotForbidden } from '../helpers/isWriteTypebotForbidden'
import { archiveResults } from '@typebot.io/results/archiveResults'
import { removeObjectsFromTypebot } from '@typebot.io/lib/s3/removeObjectsRecursively'
export const deleteTypebot = authenticatedProcedure
.meta({
@ -40,6 +41,7 @@ export const deleteTypebot = authenticatedProcedure
groups: true,
workspace: {
select: {
id: true,
isSuspended: true,
isPastDue: true,
members: {
@ -66,8 +68,10 @@ export const deleteTypebot = authenticatedProcedure
const { success } = await archiveResults(prisma)({
typebot: {
groups: existingTypebot.groups,
} as Pick<Typebot, 'groups'>,
id: typebotId,
workspaceId: existingTypebot.workspace.id,
groups: existingTypebot.groups as Typebot['groups'],
},
resultsFilter: { typebotId },
})
if (!success)
@ -82,6 +86,10 @@ export const deleteTypebot = authenticatedProcedure
where: { id: typebotId },
data: { isArchived: true, publicId: null, customDomain: null },
})
await removeObjectsFromTypebot({
workspaceId: existingTypebot.workspace.id,
typebotId,
})
return {
message: 'success',
}

View File

@ -6,6 +6,7 @@ import { TRPCError } from '@trpc/server'
import { isNotEmpty } from '@typebot.io/lib/utils'
import Stripe from 'stripe'
import { env } from '@typebot.io/env'
import { removeObjectsFromWorkspace } from '@typebot.io/lib/s3/removeObjectsRecursively'
export const deleteWorkspace = authenticatedProcedure
.meta({
@ -44,6 +45,8 @@ export const deleteWorkspace = authenticatedProcedure
where: { id: workspaceId },
})
await removeObjectsFromWorkspace(workspaceId)
if (isNotEmpty(workspace.stripeId) && env.STRIPE_SECRET_KEY) {
const stripe = new Stripe(env.STRIPE_SECRET_KEY, {
apiVersion: '2022-11-15',

View File

@ -1,32 +1,24 @@
{
"id": "cl45ojo7z01383q1av699t0qj",
"createdAt": "2022-06-08T14:22:14.879Z",
"updatedAt": "2022-06-08T16:19:32.893Z",
"icon": null,
"version": "6",
"id": "cm0kr4xle00014e0mrmqvyp1x",
"name": "My typebot",
"folderId": null,
"groups": [
"events": [
{
"id": "cl45ojo7y00013q1aaysi2o6i",
"blocks": [
{
"id": "cl45ojo7y00023q1aavrwd411",
"type": "start",
"label": "Start",
"groupId": "cl45ojo7y00013q1aaysi2o6i",
"outgoingEdgeId": "cl45ojxvc00082e6gw1xqnxpp"
"outgoingEdgeId": "cl45ojxvc00082e6gw1xqnxpp",
"graphCoordinates": { "x": 0, "y": 0 },
"type": "start"
}
],
"title": "Start",
"graphCoordinates": { "x": 0, "y": 0 }
},
"groups": [
{
"id": "cl45ojrrd00062e6g17tuu9t0",
"title": "Group #1",
"graphCoordinates": { "x": 416, "y": 98 },
"blocks": [
{
"id": "cl45ojrre00072e6gk91592pj",
"type": "text",
"groupId": "cl45ojrrd00062e6g17tuu9t0",
"content": {
"richText": [
{
@ -38,76 +30,73 @@
},
{
"id": "cl45ojzs300092e6gkno525c4",
"outgoingEdgeId": "cl45okfgz000d2e6g7z3wnqgq",
"type": "file input",
"groupId": "cl45ojrrd00062e6g17tuu9t0",
"options": {
"labels": {
"button": "Upload",
"placeholder": "<strong>\n Click to upload\n </strong> or drag and drop<br>\n (size limit: 10MB)"
},
"variableId": "vcl45ok77i000a2e6g79ye53a2",
"isMultipleAllowed": true
},
"outgoingEdgeId": "cl45okfgz000d2e6g7z3wnqgq"
"isMultipleAllowed": true,
"labels": {
"placeholder": "<strong>\n Click to upload\n </strong> or drag and drop<br>\n (size limit: 10MB)",
"button": "Upload"
}
],
"title": "Group #1",
"graphCoordinates": { "x": 416, "y": 98 }
}
}
]
},
{
"id": "cl45ok963000b2e6g2ky0wkvx",
"title": "Group #2",
"graphCoordinates": { "x": 863, "y": 249 },
"blocks": [
{
"id": "cl45ok963000c2e6g9snvbhw4",
"type": "text",
"groupId": "cl45ok963000b2e6g2ky0wkvx",
"content": {
"richText": [
{ "type": "p", "children": [{ "text": "Thank you!" }] }
]
}
}
],
"title": "Group #2",
"graphCoordinates": { "x": 863, "y": 249 }
]
}
],
"variables": [{ "id": "vcl45ok77i000a2e6g79ye53a2", "name": "Files" }],
"edges": [
{
"id": "cl45ojxvc00082e6gw1xqnxpp",
"to": { "groupId": "cl45ojrrd00062e6g17tuu9t0" },
"from": {
"blockId": "cl45ojo7y00023q1aavrwd411",
"groupId": "cl45ojo7y00013q1aaysi2o6i"
}
"from": { "eventId": "cl45ojo7y00013q1aaysi2o6i" },
"to": { "groupId": "cl45ojrrd00062e6g17tuu9t0" }
},
{
"id": "cl45okfgz000d2e6g7z3wnqgq",
"to": { "groupId": "cl45ok963000b2e6g2ky0wkvx" },
"from": {
"blockId": "cl45ojzs300092e6gkno525c4",
"groupId": "cl45ojrrd00062e6g17tuu9t0"
"from": { "blockId": "cl45ojzs300092e6gkno525c4" },
"to": { "groupId": "cl45ok963000b2e6g2ky0wkvx" }
}
],
"variables": [
{
"id": "vcl45ok77i000a2e6g79ye53a2",
"name": "Files",
"isSessionVariable": false
}
],
"theme": {
"general": { "font": "Open Sans", "background": { "type": "None" } },
"chat": {
"inputs": {
"color": "#303235",
"backgroundColor": "#FFFFFF",
"placeholderColor": "#9095A0"
},
"buttons": { "color": "#FFFFFF", "backgroundColor": "#0042DA" },
"hostAvatar": {
"url": "https://avatars.githubusercontent.com/u/16015833?v=4",
"isEnabled": true
"isEnabled": true,
"url": "https://avatars.githubusercontent.com/u/16015833?v=4"
},
"hostBubbles": { "color": "#303235", "backgroundColor": "#F7F8FF" },
"guestBubbles": { "color": "#FFFFFF", "backgroundColor": "#FF8E21" }
},
"general": { "font": "Open Sans", "background": { "type": "None" } }
"hostBubbles": { "backgroundColor": "#F7F8FF", "color": "#303235" },
"guestBubbles": { "backgroundColor": "#FF8E21", "color": "#FFFFFF" },
"buttons": { "backgroundColor": "#0042DA", "color": "#FFFFFF" },
"inputs": {
"backgroundColor": "#FFFFFF",
"color": "#303235",
"placeholderColor": "#9095A0"
}
}
},
"selectedThemeTemplateId": null,
"settings": {
"general": {
"isBrandingEnabled": true,
@ -115,12 +104,21 @@
"isHideQueryParamsEnabled": true,
"isNewResultOnRefreshEnabled": false
},
"typingEmulation": { "enabled": true, "speed": 300, "maxDelay": 1.5 },
"metadata": {
"description": "Build beautiful conversational forms and embed them directly in your applications without a line of code. Triple your response rate and collect answers that has more value compared to a traditional form."
},
"typingEmulation": { "speed": 300, "enabled": true, "maxDelay": 1.5 }
},
"publicId": "my-typebot-699t0qj",
"customDomain": null,
"workspaceId": "proWorkspace"
}
},
"createdAt": "2024-09-02T08:41:53.426Z",
"updatedAt": "2024-09-02T08:41:53.426Z",
"icon": null,
"folderId": null,
"publicId": null,
"customDomain": null,
"workspaceId": "proWorkspace",
"resultsTablePreferences": null,
"isArchived": false,
"isClosed": false,
"whatsAppCredentialsId": null,
"riskLevel": null
}

View File

@ -21,7 +21,6 @@ test('should work as expected', async ({ page, browser }) => {
getTestAsset('typebots/fileUpload.json'),
getTestAsset('typebots/hugeGroup.json'),
])
await expect(page.locator(`text="3"`)).toBeVisible()
await page.locator('text="Upload 3 files"').click()
await expect(page.locator(`text="3 files uploaded"`)).toBeVisible()
await page.goto(`${env.NEXTAUTH_URL}/typebots/${typebotId}/results`)
@ -68,6 +67,7 @@ test('should work as expected', async ({ page, browser }) => {
page.getByRole('button', { name: 'Delete' }).click()
await page.locator('button >> text="Delete"').click()
await expect(page.locator('text="api.json"')).toBeHidden()
await page2.goto(urls[0])
await expect(page2.locator('pre')).toBeHidden()
const page3 = await browser.newPage()
await page3.goto(urls[0])
await expect(page3.locator('pre')).toBeHidden()
})

View File

@ -1,43 +0,0 @@
import { env } from '@typebot.io/env'
import { Client } from 'minio'
export const deleteFilesFromBucket = async ({
urls,
}: {
urls: string[]
}): Promise<void> => {
if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
throw new Error(
'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
)
const minioClient = new Client({
endPoint: env.S3_ENDPOINT,
port: env.S3_PORT,
useSSL: env.S3_SSL,
accessKey: env.S3_ACCESS_KEY,
secretKey: env.S3_SECRET_KEY,
region: env.S3_REGION,
})
const bucket = env.S3_BUCKET
const keys = urls.reduce<string[]>(
(keys, url) => [
...keys,
...addKeyIfIncludesPublicCustomDomain(url),
...addKeyIfIncludesDefaultEndpoint(url, bucket),
],
[]
)
return minioClient.removeObjects(bucket, keys)
}
const addKeyIfIncludesPublicCustomDomain = (url: string) =>
env.S3_PUBLIC_CUSTOM_DOMAIN && url.includes(env.S3_PUBLIC_CUSTOM_DOMAIN)
? [url.split(env.S3_PUBLIC_CUSTOM_DOMAIN + '/')[1]]
: []
const addKeyIfIncludesDefaultEndpoint = (url: string, bucket: string) =>
url.includes(env.S3_ENDPOINT as string) ? [url.split(`/${bucket}/`)[1]] : []

View File

@ -1,5 +1,6 @@
import { env } from '@typebot.io/env'
import { Client, PostPolicyResult } from 'minio'
import { PostPolicyResult } from 'minio'
import { initClient } from './initClient'
type Props = {
filePath: string
@ -14,19 +15,7 @@ export const generatePresignedPostPolicy = async ({
fileType,
maxFileSize,
}: Props): Promise<PostPolicyResult> => {
if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
throw new Error(
'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
)
const minioClient = new Client({
endPoint: env.S3_ENDPOINT,
port: env.S3_PORT,
useSSL: env.S3_SSL,
accessKey: env.S3_ACCESS_KEY,
secretKey: env.S3_SECRET_KEY,
region: env.S3_REGION,
})
const minioClient = initClient()
const postPolicy = minioClient.newPostPolicy()
if (maxFileSize)

View File

@ -1,5 +1,5 @@
import { env } from '@typebot.io/env'
import { Client } from 'minio'
import { initClient } from './initClient'
type Props = {
key: string
@ -9,19 +9,6 @@ export const getFileTempUrl = async ({
key,
expires,
}: Props): Promise<string> => {
if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
throw new Error(
'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
)
const minioClient = new Client({
endPoint: env.S3_ENDPOINT,
port: env.S3_PORT,
useSSL: env.S3_SSL,
accessKey: env.S3_ACCESS_KEY,
secretKey: env.S3_SECRET_KEY,
region: env.S3_REGION,
})
const minioClient = initClient()
return minioClient.presignedGetObject(env.S3_BUCKET, key, expires ?? 3600)
}

View File

@ -1,24 +1,12 @@
import { env } from '@typebot.io/env'
import { Client } from 'minio'
import { initClient } from './initClient'
type Props = {
folderPath: string
}
export const getFolderSize = async ({ folderPath }: Props) => {
if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
throw new Error(
'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
)
const minioClient = new Client({
endPoint: env.S3_ENDPOINT,
port: env.S3_PORT,
useSSL: env.S3_SSL,
accessKey: env.S3_ACCESS_KEY,
secretKey: env.S3_SECRET_KEY,
region: env.S3_REGION,
})
const minioClient = initClient()
return new Promise<number>((resolve, reject) => {
let totalSize = 0

View File

@ -0,0 +1,20 @@
import { env } from '@typebot.io/env'
import { Client } from 'minio'
export const initClient = () => {
if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
throw new Error(
'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
)
const minioClient = new Client({
endPoint: env.S3_ENDPOINT,
port: env.S3_PORT,
useSSL: env.S3_SSL,
accessKey: env.S3_ACCESS_KEY,
secretKey: env.S3_SECRET_KEY,
region: env.S3_REGION,
})
return minioClient
}

View File

@ -0,0 +1,67 @@
import { env } from '@typebot.io/env'
import { initClient } from './initClient'
const removeObjectsRecursively = async (prefix: string) => {
const minioClient = initClient()
const bucketName = env.S3_BUCKET
const objectsStream = minioClient.listObjectsV2(bucketName, prefix, true)
for await (const obj of objectsStream) {
try {
await minioClient.removeObject(bucketName, obj.name)
} catch (err) {
console.error(`Error removing ${obj.name}:`, err)
}
}
}
export const removeObjectsFromWorkspace = async (workspaceId: string) => {
await removeObjectsRecursively(`public/workspaces/${workspaceId}/`)
await removeObjectsRecursively(`private/workspaces/${workspaceId}/`)
}
export const removeObjectsFromResult = async ({
workspaceId,
resultIds,
typebotId,
}: {
workspaceId: string
resultIds: string[]
typebotId: string
}) => {
for (const resultId of resultIds) {
await removeObjectsRecursively(
`public/workspaces/${workspaceId}/typebots/${typebotId}/results/${resultId}/`
)
}
}
export const removeAllObjectsFromResult = async ({
workspaceId,
typebotId,
}: {
workspaceId: string
typebotId: string
}) => {
await removeObjectsRecursively(
`public/workspaces/${workspaceId}/typebots/${typebotId}/results/`
)
}
export const removeObjectsFromTypebot = async ({
typebotId,
workspaceId,
}: {
typebotId: string
workspaceId: string
}) => {
await removeObjectsRecursively(
`public/workspaces/${workspaceId}/typebots/${typebotId}/`
)
}
export const removeObjectsFromUser = async (userId: string) => {
await removeObjectsRecursively(`public/users/${userId}/`)
}

View File

@ -1,5 +1,5 @@
import { env } from '@typebot.io/env'
import { Client } from 'minio'
import { initClient } from './initClient'
type Props = {
key: string
@ -12,19 +12,7 @@ export const uploadFileToBucket = async ({
file,
mimeType,
}: Props): Promise<string> => {
if (!env.S3_ENDPOINT || !env.S3_ACCESS_KEY || !env.S3_SECRET_KEY)
throw new Error(
'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
)
const minioClient = new Client({
endPoint: env.S3_ENDPOINT,
port: env.S3_PORT,
useSSL: env.S3_SSL,
accessKey: env.S3_ACCESS_KEY,
secretKey: env.S3_SECRET_KEY,
region: env.S3_REGION,
})
const minioClient = initClient()
await minioClient.putObject(env.S3_BUCKET, 'public/' + key, file, {
'Content-Type': mimeType,

View File

@ -1,11 +1,13 @@
import { Prisma, PrismaClient } from '@typebot.io/prisma'
import { Block, Typebot } from '@typebot.io/schemas'
import { deleteFilesFromBucket } from '@typebot.io/lib/s3/deleteFilesFromBucket'
import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants'
import { Typebot } from '@typebot.io/schemas'
import { isDefined } from '@typebot.io/lib'
import {
removeAllObjectsFromResult,
removeObjectsFromResult,
} from '@typebot.io/lib/s3/removeObjectsRecursively'
type ArchiveResultsProps = {
typebot: Pick<Typebot, 'groups'>
typebot: Pick<Typebot, 'groups' | 'workspaceId' | 'id'>
resultsFilter?: Omit<Prisma.ResultWhereInput, 'typebotId'> & {
typebotId: string
}
@ -15,10 +17,6 @@ export const archiveResults =
(prisma: PrismaClient) =>
async ({ typebot, resultsFilter }: ArchiveResultsProps) => {
const batchSize = 100
const fileUploadBlockIds = typebot.groups
.flatMap<Block>((group) => group.blocks)
.filter((block) => block.type === InputBlockType.FILE)
.map((block) => block.id)
let currentTotalResults = 0
@ -33,6 +31,8 @@ export const archiveResults =
let progress = 0
const isDeletingAllResults = resultsFilter?.id === undefined
do {
progress += batchSize
console.log(`Archiving ${progress} / ${resultsCount} results...`)
@ -54,19 +54,6 @@ export const archiveResults =
const resultIds = resultsToDelete.map((result) => result.id)
if (fileUploadBlockIds.length > 0) {
const filesToDelete = await prisma.answer.findMany({
where: {
resultId: { in: resultIds },
blockId: { in: fileUploadBlockIds },
},
})
if (filesToDelete.length > 0)
await deleteFilesFromBucket({
urls: filesToDelete.flatMap((a) => a.content.split(', ')),
})
}
await prisma.$transaction([
prisma.log.deleteMany({
where: {
@ -112,7 +99,21 @@ export const archiveResults =
},
}),
])
if (!isDeletingAllResults) {
await removeObjectsFromResult({
workspaceId: typebot.workspaceId,
resultIds: resultIds,
typebotId: typebot.id,
})
}
} while (currentTotalResults >= batchSize)
if (isDeletingAllResults) {
await removeAllObjectsFromResult({
workspaceId: typebot.workspaceId,
typebotId: typebot.id,
})
}
return { success: true }
}

View File

@ -1,9 +1,3 @@
import { destroyUser } from './helpers/destroyUser'
import { promptAndSetEnvironment } from './utils'
const runDestroyUser = async () => {
await promptAndSetEnvironment('production')
return destroyUser()
}
runDestroyUser()
destroyUser()

View File

@ -1,6 +1,10 @@
import { isCancel, text, confirm } from '@clack/prompts'
import { Plan, PrismaClient } from '@typebot.io/prisma'
import { PrismaClient } from '@typebot.io/prisma'
import { writeFileSync } from 'fs'
import {
removeObjectsFromUser,
removeObjectsFromWorkspace,
} from '@typebot.io/lib/s3/removeObjectsRecursively'
export const destroyUser = async (userEmail?: string) => {
const prisma = new PrismaClient()
@ -50,8 +54,16 @@ export const destroyUser = async (userEmail?: string) => {
}
console.log(
'Workspaces plans:',
workspaces.map((w) => w.plan)
'Workspaces:',
JSON.stringify(
workspaces.map((w) => ({
id: w.id,
plan: w.plan,
members: w.members,
})),
null,
2
)
)
const proceed = await confirm({ message: 'Proceed?' })
@ -61,13 +73,15 @@ export const destroyUser = async (userEmail?: string) => {
}
for (const workspace of workspaces) {
const hasResults = workspace.typebots.some((t) => t.results.length > 0)
if (hasResults) {
console.log(
`Workspace ${workspace.name} has results. Deleting results first...`,
workspace.typebots.filter((t) => t.results.length > 0)
const totalResults = workspace.typebots.reduce(
(acc, typebot) => acc + typebot.results.length,
0
)
if (totalResults > 0) {
console.log(
`Workspace ${workspace.name} has ${totalResults} results. We should delete them first...`
)
console.log(JSON.stringify({ members: workspace.members }, null, 2))
const proceed = await confirm({ message: 'Proceed?' })
if (!proceed || typeof proceed !== 'boolean') {
console.log('Aborting')
@ -82,9 +96,11 @@ export const destroyUser = async (userEmail?: string) => {
}
}
await prisma.workspace.delete({ where: { id: workspace.id } })
await removeObjectsFromWorkspace(workspace.id)
}
const user = await prisma.user.delete({ where: { email } })
await removeObjectsFromUser(user.id)
console.log(`Deleted user ${JSON.stringify(user, null, 2)}`)
console.log(`User deleted.`, JSON.stringify(user, null, 2))
}

View File

@ -20,7 +20,7 @@
"insertUsersInBrevoList": "tsx insertUsersInBrevoList.ts",
"getUsage": "tsx getUsage.ts",
"suspendWorkspace": "tsx suspendWorkspace.ts",
"destroyUser": "tsx destroyUser.ts",
"destroyUser": "SKIP_ENV_CHECK=true dotenv -e ./.env.production -- tsx destroyUser.ts",
"updateTypebot": "tsx updateTypebot.ts",
"updateWorkspace": "tsx updateWorkspace.ts",
"inspectTypebot": "tsx inspectTypebot.ts",