🔧 (scripts) Add typebot fix script

Closes #192

.gitignore

@@ -31,4 +31,5 @@ dump.tar
 
 __env.js
 
-invalidTypebots.json
+typebotsToFix.json
+**/scripts/logs

apps/builder/public/templates/lead-gen.json

@@ -111,7 +111,8 @@
           "groupId": "o4SH1UtKANnW5N5D67oZUz",
           "options": {
             "labels": { "button": "Send", "placeholder": "Type your email..." },
-            "variableId": "v3VFChNVSCXQ2rXv4DrJ8Ah"
+            "variableId": "v3VFChNVSCXQ2rXv4DrJ8Ah",
+            "retryMessageContent": "This email doesn't seem to be valid. Can you type it again?"
           },
           "outgoingEdgeId": "w3MiN1Ct38jT5NykVsgmb5"
         }

apps/builder/src/pages/api/typebots/[typebotId].ts

@@ -5,6 +5,8 @@ import { canReadTypebots, canWriteTypebots } from '@/utils/api/dbRules'
 import { methodNotAllowed, notAuthenticated } from 'utils/api'
 import { getAuthenticatedUser } from '@/features/auth/api'
 import { archiveResults } from '@/features/results/api'
+import { typebotSchema } from 'models'
+import { captureEvent } from '@sentry/nextjs'
 
 const handler = async (req: NextApiRequest, res: NextApiResponse) => {
   const user = await getAuthenticatedUser(req)
@@ -55,6 +57,16 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => {
   }
   if (req.method === 'PUT') {
     const data = typeof req.body === 'string' ? JSON.parse(req.body) : req.body
+    const parser = typebotSchema.safeParse(data)
+    if ('error' in parser) {
+      captureEvent({
+        message: 'Typebot schema validation failed',
+        extra: {
+          typebotId: data.id,
+          error: parser.error,
+        },
+      })
+    }
     const existingTypebot = await prisma.typebot.findFirst({
       where: canReadTypebots(typebotId, user),
       select: { updatedAt: true },

packages/models/src/features/blocks/inputs/email.ts

@@ -11,8 +11,7 @@ export const emailInputOptionsSchema = optionBaseSchema
   .and(textInputOptionsBaseSchema)
   .and(
     z.object({
-      // TODO: make it required once database migration is done
-      retryMessageContent: z.string().optional(),
+      retryMessageContent: z.string(),
     })
   )
 

packages/scripts/fixTypebots.ts

@@ -0,0 +1,195 @@
+import { PrismaClient } from 'db'
+import { readFileSync, writeFileSync } from 'fs'
+import {
+  Block,
+  BlockOptions,
+  BlockType,
+  defaultEmailInputOptions,
+  Group,
+  InputBlockType,
+  Theme,
+  Typebot,
+  typebotSchema,
+} from 'models'
+import { isNotDefined } from 'utils'
+import { promptAndSetEnvironment } from './utils'
+import { detailedDiff } from 'deep-object-diff'
+
+const fixTypebot = (brokenTypebot: Typebot) =>
+  ({
+    ...brokenTypebot,
+    theme: fixTheme(brokenTypebot.theme),
+    groups: fixGroups(brokenTypebot.groups),
+  } satisfies Typebot)
+
+const fixTheme = (brokenTheme: Theme) =>
+  ({
+    ...brokenTheme,
+    chat: {
+      ...brokenTheme.chat,
+      hostAvatar: brokenTheme.chat.hostAvatar
+        ? {
+            isEnabled: brokenTheme.chat.hostAvatar.isEnabled,
+            url: brokenTheme.chat.hostAvatar.url ?? undefined,
+          }
+        : undefined,
+    },
+  } satisfies Theme)
+
+const fixGroups = (brokenGroups: Group[]) =>
+  brokenGroups.map(
+    (brokenGroup, index) =>
+      ({
+        ...brokenGroup,
+        graphCoordinates: {
+          ...brokenGroup.graphCoordinates,
+          x: brokenGroup.graphCoordinates.x ?? 0,
+          y: brokenGroup.graphCoordinates.y ?? 0,
+        },
+        blocks: fixBlocks(brokenGroup.blocks, brokenGroup.id, index),
+      } satisfies Group)
+  )
+
+const fixBlocks = (
+  brokenBlocks: Block[],
+  groupId: string,
+  groupIndex: number
+) => {
+  if (groupIndex === 0 && brokenBlocks.length > 1) return [brokenBlocks[0]]
+  return brokenBlocks
+    .filter((block) => block && Object.keys(block).length > 0)
+    .map((brokenBlock) => {
+      return removeUndefinedFromObject({
+        ...brokenBlock,
+        webhookId:
+          ('webhookId' in brokenBlock ? brokenBlock.webhookId : undefined) ??
+          ('webhook' in brokenBlock && brokenBlock.webhook
+            ? //@ts-ignore
+              brokenBlock.webhook.id
+            : undefined),
+        webhook: undefined,
+        groupId: brokenBlock.groupId ?? groupId,
+        options:
+          brokenBlock && 'options' in brokenBlock && brokenBlock.options
+            ? fixBrokenBlockOption(brokenBlock.options, brokenBlock.type)
+            : undefined,
+      })
+    }) as Block[]
+}
+
+const fixBrokenBlockOption = (option: BlockOptions, blockType: BlockType) =>
+  removeUndefinedFromObject({
+    ...option,
+    sheetId:
+      'sheetId' in option && option.sheetId
+        ? option.sheetId.toString()
+        : undefined,
+    step: 'step' in option && option.step ? option.step : undefined,
+    value: 'value' in option && option.value ? option.value : undefined,
+    retryMessageContent: fixRetryMessageContent(
+      //@ts-ignore
+      option.retryMessageContent,
+      blockType
+    ),
+  }) as BlockOptions
+
+const fixRetryMessageContent = (
+  retryMessageContent: string | undefined,
+  blockType: BlockType
+) => {
+  if (isNotDefined(retryMessageContent) && blockType === InputBlockType.EMAIL)
+    return defaultEmailInputOptions.retryMessageContent
+  if (isNotDefined(retryMessageContent)) return undefined
+  return retryMessageContent
+}
+
+const removeUndefinedFromObject = (obj: any) => {
+  Object.keys(obj).forEach((key) => obj[key] === undefined && delete obj[key])
+  return obj
+}
+
+const resolve = (path: string, obj: object, separator = '.') => {
+  const properties = Array.isArray(path) ? path : path.split(separator)
+  //@ts-ignore
+  return properties.reduce((prev, curr) => prev?.[curr], obj)
+}
+
+const fixTypebots = async () => {
+  await promptAndSetEnvironment()
+  const prisma = new PrismaClient({
+    log: [{ emit: 'event', level: 'query' }, 'info', 'warn', 'error'],
+  })
+
+  prisma.$on('query', (e) => {
+    console.log(e.query)
+    console.log(e.params)
+    console.log(e.duration, 'ms')
+  })
+
+  const typebots = JSON.parse(readFileSync('typebots.json', 'utf-8')) as any[]
+
+  const total = typebots.length
+  let totalFixed = 0
+  let progress = 0
+  const fixedTypebots: Typebot[] = []
+  const diffs: any[] = []
+  for (const typebot of typebots) {
+    progress += 1
+    console.log(
+      `Progress: ${progress}/${total} (${Math.round(
+        (progress / total) * 100
+      )}%) (${totalFixed} fixed typebots)`
+    )
+    const parser = typebotSchema.safeParse({
+      ...typebot,
+      updatedAt: new Date(typebot.updatedAt),
+      createdAt: new Date(typebot.createdAt),
+    })
+    if ('error' in parser) {
+      const fixedTypebot = {
+        ...fixTypebot(typebot),
+        updatedAt: new Date(typebot.updatedAt),
+        createdAt: new Date(typebot.createdAt),
+      }
+      fixedTypebots.push(fixedTypebot)
+      totalFixed += 1
+      diffs.push({
+        id: typebot.id,
+        failedObject: resolve(
+          parser.error.issues[0].path.join('.'),
+          fixedTypebot
+        ),
+        ...detailedDiff(typebot, fixedTypebot),
+      })
+    }
+  }
+  writeFileSync('logs/fixedTypebots.json', JSON.stringify(fixedTypebots))
+  writeFileSync(
+    'logs/diffs.json',
+    JSON.stringify(diffs.reverse().slice(0, 100))
+  )
+}
+
+// export const parseZodError = (parser: any) => {
+//   if ('error' in parser) {
+//     console.log(
+//       parser.error.issues.map((issue) =>
+//         JSON.stringify({
+//           message: issue.message,
+//           path: issue.path,
+//         })
+//       )
+//     )
+//     writeFileSync(
+//       'failedObject.json',
+//       JSON.stringify(
+//         resolve(parser.error.issues[0].path.join('.'), fixedTypebot)
+//       )
+//     )
+//     writeFileSync('failedTypebot.json', JSON.stringify(fixedTypebot))
+//     writeFileSync('issue.json', JSON.stringify(parser.error.issues))
+//     exit()
+//   }
+// }
+
+fixTypebots()

packages/scripts/package.json

@@ -9,12 +9,14 @@
     "db:backup": "tsx backupDatabase.ts",
     "db:restore": "tsx restoreDatabase.ts",
     "db:setCustomPlan": "tsx setCustomPlan.ts",
-    "db:bulkUpdate": "tsx bulkUpdate.ts"
+    "db:bulkUpdate": "tsx bulkUpdate.ts",
+    "db:fixTypebots": "tsx fixTypebots.ts"
   },
   "devDependencies": {
     "@types/node": "18.11.17",
     "@types/prompts": "^2.4.2",
     "db": "workspace:*",
+    "deep-object-diff": "1.1.9",
     "emails": "workspace:*",
     "got": "12.5.3",
     "models": "workspace:*",

pnpm-lock.yaml

@@ -679,6 +679,7 @@ importers:
       '@types/node': 18.11.17
       '@types/prompts': ^2.4.2
       db: workspace:*
+      deep-object-diff: 1.1.9
       emails: workspace:*
       got: 12.5.3
       models: workspace:*
@@ -692,6 +693,7 @@ importers:
       '@types/node': 18.11.17
       '@types/prompts': 2.4.2
       db: link:../db
+      deep-object-diff: 1.1.9
       emails: link:../emails
       got: 12.5.3
       models: link:../models
@@ -9317,7 +9319,6 @@ packages:
 
   /deep-object-diff/1.1.9:
     resolution: {integrity: sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==}
-    dev: false
 
   /deepmerge/4.2.2:
     resolution: {integrity: sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==}