⚡ (whatsapp) Improve / fix markdown serializer

Forked remark-slate code to create Typebot's own serializer

Closes #1056

apps/builder/public/templates/basic-chat-gpt.json

@@ -92,7 +92,6 @@
                 "dialogueVariableId": "vabkycu0qqff5d6ar2ama16pf"
               }
             ],
-            "credentialsId": "clpjnjrbt00051aliw6610w1z",
             "responseMapping": [
               {
                 "item": "Message content",
@@ -101,6 +100,16 @@
             ]
           }
         },
+        {
+          "id": "myldn1l1nfdwwm8qvza71rwv",
+          "type": "text",
+          "content": {
+            "richText": [
+              { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] }
+            ]
+          },
+          "groupId": "a6ymhjwtkqwp8t127plz8qmk"
+        },
         {
           "id": "yblc864bzipaqfja7b2o3oo0",
           "type": "Set variable",
@@ -108,17 +117,8 @@
             "variableId": "vabkycu0qqff5d6ar2ama16pf",
             "type": "Append value(s)",
             "item": "{{Assistant Message}}"
-          }
+          },
-        },
+          "outgoingEdgeId": "at8takz56suqmaul5teazymb"
-        {
-          "id": "myldn1l1nfdwwm8qvza71rwv",
-          "outgoingEdgeId": "y8ml9ljnsydol9b42fd9zdve",
-          "type": "text",
-          "content": {
-            "richText": [
-              { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] }
-            ]
-          }
         }
       ]
     },
@@ -155,20 +155,23 @@
       "from": { "blockId": "gphm5wy1md9cunwkdtbzg6nq" },
       "to": { "groupId": "qfrz5nwm63g12dajsjxothb5" }
     },
-    {
-      "id": "y8ml9ljnsydol9b42fd9zdve",
-      "from": { "blockId": "myldn1l1nfdwwm8qvza71rwv" },
-      "to": { "groupId": "qfrz5nwm63g12dajsjxothb5" }
-    },
     {
       "id": "fpj0xacppqd1s5slyljzhzc9",
       "from": { "blockId": "m4jadtknjb3za3gvxj1xdn1k" },
       "to": { "groupId": "a6ymhjwtkqwp8t127plz8qmk" }
     },
     {
+      "id": "pn7omb9mx5xxc4mzq028fcmq",
       "from": { "eventId": "ewnfbo0exlu7ihfu2lu2lusm" },
-      "to": { "groupId": "t3tv4dm3khwmiotjle5jb65g" },
+      "to": { "groupId": "t3tv4dm3khwmiotjle5jb65g" }
-      "id": "pn7omb9mx5xxc4mzq028fcmq"
+    },
+    {
+      "from": {
+        "blockId": "yblc864bzipaqfja7b2o3oo0",
+        "groupId": "a6ymhjwtkqwp8t127plz8qmk"
+      },
+      "to": { "groupId": "qfrz5nwm63g12dajsjxothb5" },
+      "id": "at8takz56suqmaul5teazymb"
     }
   ],
   "variables": [

apps/builder/public/templates/chat-gpt-personas.json

@@ -74,7 +74,6 @@
                 "dialogueVariableId": "vu9adij5penetej2xz89htfe6"
               }
             ],
-            "credentialsId": "clpjnjrbt00051aliw6610w1z",
             "responseMapping": [
               {
                 "item": "Message content",
@@ -83,15 +82,6 @@
             ]
           }
         },
-        {
-          "id": "kftq9x1wnrcefzc268ydmqkn",
-          "type": "Set variable",
-          "options": {
-            "variableId": "vu9adij5penetej2xz89htfe6",
-            "type": "Append value(s)",
-            "item": "{{Assistant Message}}"
-          }
-        },
         {
           "id": "myldn1l1nfdwwm8qvza71rwv",
           "type": "text",
@@ -99,6 +89,16 @@
             "richText": [
               { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] }
             ]
+          },
+          "groupId": "dmg57mgick51p8l5pnyqtyf9"
+        },
+        {
+          "id": "kftq9x1wnrcefzc268ydmqkn",
+          "type": "Set variable",
+          "options": {
+            "variableId": "vu9adij5penetej2xz89htfe6",
+            "type": "Append value(s)",
+            "item": "{{Assistant Message}}"
           }
         },
         {
@@ -139,7 +139,6 @@
                 "dialogueVariableId": "vu9adij5penetej2xz89htfe6"
               }
             ],
-            "credentialsId": "clpjnjrbt00051aliw6610w1z",
             "responseMapping": [
               {
                 "item": "Message content",
@@ -148,15 +147,6 @@
             ]
           }
         },
-        {
-          "id": "dyfigmu4095x8p99qe461zbh",
-          "type": "Set variable",
-          "options": {
-            "variableId": "vu9adij5penetej2xz89htfe6",
-            "type": "Append value(s)",
-            "item": "{{Assistant Message}}"
-          }
-        },
         {
           "id": "sei88rrjcmpgm3vhxjvkofyt",
           "type": "text",
@@ -164,6 +154,16 @@
             "richText": [
               { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] }
             ]
+          },
+          "groupId": "fj5z2nx488htv0843kq6qeyk"
+        },
+        {
+          "id": "dyfigmu4095x8p99qe461zbh",
+          "type": "Set variable",
+          "options": {
+            "variableId": "vu9adij5penetej2xz89htfe6",
+            "type": "Append value(s)",
+            "item": "{{Assistant Message}}"
           }
         },
         {
@@ -204,7 +204,6 @@
                 "dialogueVariableId": "vu9adij5penetej2xz89htfe6"
               }
             ],
-            "credentialsId": "clpjnjrbt00051aliw6610w1z",
             "responseMapping": [
               {
                 "item": "Message content",
@@ -213,15 +212,6 @@
             ]
           }
         },
-        {
-          "id": "usolgxcte60rin18ccygzbdu",
-          "type": "Set variable",
-          "options": {
-            "variableId": "vu9adij5penetej2xz89htfe6",
-            "type": "Append value(s)",
-            "item": "{{Assistant Message}}"
-          }
-        },
         {
           "id": "h96lile0evtqa0jx24gmfo25",
           "type": "text",
@@ -229,6 +219,16 @@
             "richText": [
               { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] }
             ]
+          },
+          "groupId": "csbysu8dr08zxr4i6hzvzjdf"
+        },
+        {
+          "id": "usolgxcte60rin18ccygzbdu",
+          "type": "Set variable",
+          "options": {
+            "variableId": "vu9adij5penetej2xz89htfe6",
+            "type": "Append value(s)",
+            "item": "{{Assistant Message}}"
           }
         },
         {
@@ -678,9 +678,9 @@
       "to": { "groupId": "dmg57mgick51p8l5pnyqtyf9" }
     },
     {
+      "id": "k5bj58emklqfqv3hemko4u23",
       "from": { "eventId": "w99qhdr20tw02sfrfwkfc1tg" },
-      "to": { "groupId": "bofjp88arodr4k0btv2esyqy" },
+      "to": { "groupId": "bofjp88arodr4k0btv2esyqy" }
-      "id": "k5bj58emklqfqv3hemko4u23"
     }
   ],
   "variables": [

packages/bot-engine/package.json

@@ -30,14 +30,13 @@
     "nodemailer": "6.9.3",
     "openai": "4.19.0",
     "qs": "6.11.2",
-    "remark-slate": "1.8.6",
     "stripe": "12.13.0"
   },
   "devDependencies": {
-    "@types/nodemailer": "6.4.8",
-    "@types/qs": "6.9.7",
-    "@typebot.io/forge-schemas": "workspace:*",
     "@typebot.io/forge": "workspace:*",
-    "@typebot.io/forge-repository": "workspace:*"
+    "@typebot.io/forge-repository": "workspace:*",
+    "@typebot.io/forge-schemas": "workspace:*",
+    "@types/nodemailer": "6.4.8",
+    "@types/qs": "6.9.7"
   }
 }

packages/bot-engine/whatsapp/convertInputToWhatsAppMessage.ts

@@ -1,11 +1,11 @@
 import { ButtonItem, ContinueChatResponse } from '@typebot.io/schemas'
 import { WhatsAppSendingMessage } from '@typebot.io/schemas/features/whatsapp'
-import { convertRichTextToWhatsAppText } from './convertRichTextToWhatsAppText'
 import { isDefined, isEmpty } from '@typebot.io/lib/utils'
 import { BubbleBlockType } from '@typebot.io/schemas/features/blocks/bubbles/constants'
 import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants'
 import { defaultPictureChoiceOptions } from '@typebot.io/schemas/features/blocks/inputs/pictureChoice/constants'
 import { defaultChoiceInputOptions } from '@typebot.io/schemas/features/blocks/inputs/choice/constants'
+import { convertRichTextToMarkdown } from '@typebot.io/lib/serializer/convertRichTextToMarkdown'
 
 export const convertInputToWhatsAppMessages = (
   input: NonNullable<ContinueChatResponse['input']>,
@@ -13,7 +13,9 @@ export const convertInputToWhatsAppMessages = (
 ): WhatsAppSendingMessage[] => {
   const lastMessageText =
     lastMessage?.type === BubbleBlockType.TEXT
-      ? convertRichTextToWhatsAppText(lastMessage.content.richText ?? [])
+      ? convertRichTextToMarkdown(lastMessage.content.richText ?? [], {
+          flavour: 'whatsapp',
+        })
       : undefined
   switch (input.type) {
     case InputBlockType.DATE:

packages/bot-engine/whatsapp/convertMessageToWhatsAppMessage.ts

@@ -1,9 +1,9 @@
 import { ContinueChatResponse } from '@typebot.io/schemas'
 import { WhatsAppSendingMessage } from '@typebot.io/schemas/features/whatsapp'
-import { convertRichTextToWhatsAppText } from './convertRichTextToWhatsAppText'
 import { isSvgSrc } from '@typebot.io/lib/utils'
 import { BubbleBlockType } from '@typebot.io/schemas/features/blocks/bubbles/constants'
 import { VideoBubbleContentType } from '@typebot.io/schemas/features/blocks/bubbles/video/constants'
+import { convertRichTextToMarkdown } from '@typebot.io/lib/serializer/convertRichTextToMarkdown'
 
 const mp4HttpsUrlRegex = /^https:\/\/.*\.mp4$/
 
@@ -17,7 +17,9 @@ export const convertMessageToWhatsAppMessage = (
       return {
         type: 'text',
         text: {
-          body: convertRichTextToWhatsAppText(message.content.richText),
+          body: convertRichTextToMarkdown(message.content.richText, {
+            flavour: 'whatsapp',
+          }),
         },
       }
     }

packages/bot-engine/whatsapp/convertRichTextToWhatsAppText.ts

@@ -1,9 +0,0 @@
-import { TElement } from '@udecode/plate-common'
-import { serialize } from 'remark-slate'
-
-export const convertRichTextToWhatsAppText = (richText: TElement[]): string =>
-  richText
-    .map((chunk) =>
-      serialize(chunk)?.replaceAll('**', '*').replaceAll(''', "'")
-    )
-    .join('\n')

packages/lib/package.json

@@ -9,14 +9,15 @@
     "@paralleldrive/cuid2": "2.2.1",
     "@playwright/test": "1.36.0",
     "@typebot.io/env": "workspace:*",
+    "@typebot.io/forge-repository": "workspace:*",
     "@typebot.io/prisma": "workspace:*",
     "@typebot.io/schemas": "workspace:*",
     "@typebot.io/tsconfig": "workspace:*",
+    "@types/escape-html": "^1.0.4",
     "@types/nodemailer": "6.4.8",
     "next": "14.0.3",
     "nodemailer": "6.9.3",
-    "typescript": "5.3.2",
+    "typescript": "5.3.2"
-    "@typebot.io/forge-repository": "workspace:*"
   },
   "peerDependencies": {
     "next": "14.0.0",
@@ -26,10 +27,10 @@
     "@sentry/nextjs": "7.77.0",
     "@trpc/server": "10.40.0",
     "@udecode/plate-common": "21.1.5",
+    "escape-html": "^1.0.3",
     "google-auth-library": "8.9.0",
     "got": "12.6.0",
     "minio": "7.1.3",
-    "remark-slate": "1.8.6",
     "stripe": "12.13.0",
     "zod": "3.22.4"
   }

packages/lib/serializer/ast-types.ts

@@ -0,0 +1,243 @@
+export interface NodeTypes {
+  paragraph: string
+  blockquote: string
+  code_block: string
+  link: string
+  ul_list: string
+  ol_list: string
+  listItem: string
+  listItemChild: string
+  heading: {
+    1: string
+    2: string
+    3: string
+    4: string
+    5: string
+    6: string
+  }
+  emphasis_mark: string
+  strong_mark: string
+  delete_mark: string
+  inline_code_mark: string
+  thematic_break: string
+  image: string
+  variable: string
+  ['inline-variable']: string
+}
+
+export type MdastNodeType =
+  | 'paragraph'
+  | 'heading'
+  | 'list'
+  | 'listItem'
+  | 'link'
+  | 'image'
+  | 'blockquote'
+  | 'code'
+  | 'html'
+  | 'emphasis'
+  | 'strong'
+  | 'delete'
+  | 'inlineCode'
+  | 'thematicBreak'
+  | 'text'
+
+export const defaultNodeTypes: NodeTypes = {
+  paragraph: 'p',
+  blockquote: 'blockquote',
+  code_block: 'code_block',
+  link: 'a',
+  ul_list: 'ul',
+  ol_list: 'ol',
+  listItem: 'li',
+  listItemChild: 'lic',
+  heading: {
+    1: 'h1',
+    2: 'h2',
+    3: 'h3',
+    4: 'h4',
+    5: 'h5',
+    6: 'h6',
+  },
+  emphasis_mark: 'italic',
+  strong_mark: 'bold',
+  delete_mark: 'strikeThrough',
+  inline_code_mark: 'code',
+  thematic_break: 'thematic_break',
+  image: 'image',
+  variable: 'variable',
+  ['inline-variable']: 'inline-variable',
+}
+
+export interface LeafType {
+  text: string
+  strikeThrough?: boolean
+  bold?: boolean
+  italic?: boolean
+  code?: boolean
+  parentType?: string
+}
+
+export interface BlockType {
+  type: string
+  parentType?: string
+  link?: string
+  caption?: string
+  language?: string
+  break?: boolean
+  listIndex?: number
+  children: Array<BlockType | LeafType>
+}
+
+export interface InputNodeTypes {
+  paragraph: string
+  block_quote: string
+  code_block: string
+  link: string
+  ul_list: string
+  ol_list: string
+  listItem: string
+  heading: {
+    1: string
+    2: string
+    3: string
+    4: string
+    5: string
+    6: string
+  }
+  emphasis_mark: string
+  strong_mark: string
+  delete_mark: string
+  inline_code_mark: string
+  thematic_break: string
+  image: string
+}
+
+type RecursivePartial<T> = {
+  [P in keyof T]?: RecursivePartial<T[P]>
+}
+
+export interface OptionType<T extends InputNodeTypes = InputNodeTypes> {
+  nodeTypes?: RecursivePartial<T>
+  linkDestinationKey?: string
+  imageSourceKey?: string
+  imageCaptionKey?: string
+}
+
+export interface MdastNode {
+  type?: MdastNodeType
+  ordered?: boolean
+  value?: string
+  text?: string
+  children?: Array<MdastNode>
+  depth?: 1 | 2 | 3 | 4 | 5 | 6
+  url?: string
+  alt?: string
+  lang?: string
+  // mdast metadata
+  position?: any
+  spread?: any
+  checked?: any
+  indent?: any
+}
+
+export type TextNode = { text?: string | undefined }
+
+export type CodeBlockNode<T extends InputNodeTypes> = {
+  type: T['code_block']
+  language: string | undefined
+  children: Array<TextNode>
+}
+
+export type HeadingNode<T extends InputNodeTypes> = {
+  type:
+    | T['heading'][1]
+    | T['heading'][2]
+    | T['heading'][3]
+    | T['heading'][4]
+    | T['heading'][5]
+    | T['heading'][6]
+  children: Array<DeserializedNode<T>>
+}
+
+export type ListNode<T extends InputNodeTypes> = {
+  type: T['ol_list'] | T['ul_list']
+  children: Array<DeserializedNode<T>>
+}
+
+export type ListItemNode<T extends InputNodeTypes> = {
+  type: T['listItem']
+  children: Array<DeserializedNode<T>>
+}
+
+export type ParagraphNode<T extends InputNodeTypes> = {
+  type: T['paragraph']
+  break?: true
+  children: Array<DeserializedNode<T>>
+}
+
+export type LinkNode<T extends InputNodeTypes> = {
+  type: T['link']
+  children: Array<DeserializedNode<T>>
+  [urlKey: string]: string | undefined | Array<DeserializedNode<T>>
+}
+
+export type ImageNode<T extends InputNodeTypes> = {
+  type: T['image']
+  children: Array<DeserializedNode<T>>
+  [sourceOrCaptionKey: string]: string | undefined | Array<DeserializedNode<T>>
+}
+
+export type BlockQuoteNode<T extends InputNodeTypes> = {
+  type: T['block_quote']
+  children: Array<DeserializedNode<T>>
+}
+
+export type InlineCodeMarkNode<T extends InputNodeTypes> = {
+  type: T['inline_code_mark']
+  children: Array<TextNode>
+  language: string | undefined
+}
+
+export type ThematicBreakNode<T extends InputNodeTypes> = {
+  type: T['thematic_break']
+  children: Array<DeserializedNode<T>>
+}
+
+export type ItalicNode<T extends InputNodeTypes> = {
+  [K in T['emphasis_mark']]: true
+} & {
+  children: TextNode
+}
+
+export type BoldNode = {
+  bold: true
+  children: TextNode
+}
+
+export type StrikeThoughNode = {
+  strikeThrough: true
+  children: TextNode
+}
+
+export type InlineCodeNode = {
+  code: true
+  text: string | undefined
+}
+
+export type DeserializedNode<T extends InputNodeTypes> =
+  | CodeBlockNode<T>
+  | HeadingNode<T>
+  | ListNode<T>
+  | ListItemNode<T>
+  | ParagraphNode<T>
+  | LinkNode<T>
+  | ImageNode<T>
+  | BlockQuoteNode<T>
+  | InlineCodeMarkNode<T>
+  | ThematicBreakNode<T>
+  | ItalicNode<T>
+  | BoldNode
+  | StrikeThoughNode
+  | InlineCodeNode
+  | TextNode

packages/lib/serializer/convertRichTextToMarkdown.ts

@@ -0,0 +1,41 @@
+import { TElement } from '@udecode/plate-common'
+import serialize from './serialize'
+import { defaultNodeTypes } from './ast-types'
+
+export const convertRichTextToMarkdown = (
+  richText: TElement[],
+  options?: { flavour?: 'common' | 'whatsapp' }
+) => {
+  let extraNewLinesCount = 0
+  const test = richText
+    .reduce<string[]>((acc, node) => {
+      if (node.type === 'variable' || node.type === 'inline-variable') {
+        return [
+          ...acc,
+          ...node.children.map(
+            (child) =>
+              serialize(child, {
+                nodeTypes: defaultNodeTypes,
+                flavour: options?.flavour,
+              }) as string
+          ),
+        ]
+      }
+      const serializedElement = serialize(node, {
+        nodeTypes: defaultNodeTypes,
+        flavour: options?.flavour,
+      })
+      if (!serializedElement || serializedElement === '<br>\n\n') {
+        if (extraNewLinesCount > 0) {
+          return [...acc, '']
+        }
+        extraNewLinesCount++
+        return acc
+      }
+      extraNewLinesCount = 0
+      return [...acc, serializedElement]
+    }, [])
+    .join('\n')
+
+  return test.endsWith('\n') ? test.slice(0, -2) : test
+}

packages/lib/serializer/serialize.ts

@@ -0,0 +1,266 @@
+import { BlockType, defaultNodeTypes, LeafType, NodeTypes } from './ast-types'
+import escapeHtml from 'escape-html'
+
+interface Options {
+  nodeTypes: NodeTypes
+  flavour?: 'common' | 'whatsapp'
+  listDepth?: number
+  ignoreParagraphNewline?: boolean
+}
+
+const isLeafNode = (node: BlockType | LeafType): node is LeafType => {
+  return typeof (node as LeafType).text === 'string'
+}
+
+const VOID_ELEMENTS: Array<keyof NodeTypes> = ['thematic_break', 'image']
+
+const BREAK_TAG = '<br>'
+
+export default function serialize(
+  chunk: BlockType | LeafType,
+  opts: Options = {
+    nodeTypes: defaultNodeTypes,
+  }
+) {
+  const {
+    nodeTypes: userNodeTypes = defaultNodeTypes,
+    ignoreParagraphNewline = false,
+    listDepth = -1,
+  } = opts
+
+  let text = (chunk as LeafType).text || ''
+  let type = (chunk as BlockType).type || ''
+
+  const nodeTypes: NodeTypes = {
+    ...defaultNodeTypes,
+    ...userNodeTypes,
+    heading: {
+      ...defaultNodeTypes.heading,
+      ...userNodeTypes.heading,
+    },
+  }
+
+  const LIST_TYPES = [nodeTypes.ul_list, nodeTypes.ol_list]
+
+  let children = text
+
+  if (!isLeafNode(chunk)) {
+    children = chunk.children
+      .map((c: BlockType | LeafType, index) => {
+        const isList = !isLeafNode(c)
+          ? (LIST_TYPES as string[]).includes(c.type || '')
+          : false
+
+        const selfIsList = (LIST_TYPES as string[]).includes(chunk.type || '')
+
+        // Links can have the following shape
+        // In which case we don't want to surround
+        // with break tags
+        // {
+        //  type: 'paragraph',
+        //  children: [
+        //    { text: '' },
+        //    { type: 'link', children: [{ text: foo.com }]}
+        //    { text: '' }
+        //  ]
+        // }
+        let childrenHasLink = false
+
+        if (!isLeafNode(chunk) && Array.isArray(chunk.children)) {
+          childrenHasLink = chunk.children.some(
+            (f) => !isLeafNode(f) && f.type === nodeTypes.link
+          )
+        }
+
+        return serialize(
+          {
+            ...c,
+            parentType: type === nodeTypes.listItem ? chunk.parentType : type,
+            listIndex:
+              type === nodeTypes.listItem ? chunk.listIndex : index + 1,
+          },
+          {
+            flavour: opts.flavour,
+            nodeTypes,
+            // WOAH.
+            // what we're doing here is pretty tricky, it relates to the block below where
+            // we check for ignoreParagraphNewline and set type to paragraph.
+            // We want to strip out empty paragraphs sometimes, but other times we don't.
+            // If we're the descendant of a list, we know we don't want a bunch
+            // of whitespace. If we're parallel to a link we also don't want
+            // to respect neighboring paragraphs
+            ignoreParagraphNewline:
+              (ignoreParagraphNewline ||
+                isList ||
+                selfIsList ||
+                childrenHasLink) &&
+              // if we have c.break, never ignore empty paragraph new line
+              !(c as BlockType).break,
+
+            // track depth of nested lists so we can add proper spacing
+            listDepth: (LIST_TYPES as string[]).includes(
+              (c as BlockType).type || ''
+            )
+              ? listDepth + 1
+              : listDepth,
+          }
+        )
+      })
+      .join('')
+  }
+
+  // This is pretty fragile code, check the long comment where we iterate over children
+  if (
+    !ignoreParagraphNewline &&
+    (text === '' || text === '\n') &&
+    chunk.parentType === nodeTypes.paragraph
+  ) {
+    type = nodeTypes.paragraph
+    children = BREAK_TAG
+  }
+
+  if (children === '' && !VOID_ELEMENTS.find((k) => nodeTypes[k] === type))
+    return
+
+  // Never allow decorating break tags with rich text formatting,
+  // this can malform generated markdown
+  // Also ensure we're only ever applying text formatting to leaf node
+  // level chunks, otherwise we can end up in a situation where
+  // we try applying formatting like to a node like this:
+  // "Text foo bar **baz**" resulting in "**Text foo bar **baz****"
+  // which is invalid markup and can mess everything up
+  if (children !== BREAK_TAG && isLeafNode(chunk)) {
+    if (chunk.strikeThrough && chunk.bold && chunk.italic) {
+      if (opts.flavour === 'whatsapp') {
+        children = retainWhitespaceAndFormat(children, '*_~')
+      } else {
+        children = retainWhitespaceAndFormat(children, '~~***')
+      }
+    } else if (chunk.bold && chunk.italic) {
+      if (opts.flavour === 'whatsapp') {
+        children = retainWhitespaceAndFormat(children, '*_')
+      } else {
+        children = retainWhitespaceAndFormat(children, '***')
+      }
+    } else {
+      if (chunk.bold) {
+        if (opts.flavour === 'whatsapp') {
+          children = retainWhitespaceAndFormat(children, '*')
+        } else {
+          children = retainWhitespaceAndFormat(children, '**')
+        }
+      }
+
+      if (chunk.italic) {
+        children = retainWhitespaceAndFormat(children, '_')
+      }
+
+      if (chunk.strikeThrough) {
+        children = retainWhitespaceAndFormat(children, '~~')
+      }
+
+      if (chunk.code) {
+        if (opts.flavour === 'whatsapp') {
+          children = retainWhitespaceAndFormat(children, '```')
+        } else {
+          children = retainWhitespaceAndFormat(children, '`')
+        }
+      }
+    }
+  }
+
+  switch (type) {
+    case nodeTypes.heading[1]:
+      return `# ${children}\n`
+    case nodeTypes.heading[2]:
+      return `## ${children}\n`
+    case nodeTypes.heading[3]:
+      return `### ${children}\n`
+    case nodeTypes.heading[4]:
+      return `#### ${children}\n`
+    case nodeTypes.heading[5]:
+      return `##### ${children}\n`
+    case nodeTypes.heading[6]:
+      return `###### ${children}\n`
+
+    case nodeTypes.blockquote:
+      return `> ${children}\n`
+
+    case nodeTypes.code_block:
+      return `\`\`\`${
+        (chunk as BlockType).language || ''
+      }\n${children}\n\`\`\`\n`
+
+    case nodeTypes.link:
+      return `[${children}](${(chunk as any).url || ''})`
+    case nodeTypes.image:
+      return `![${(chunk as BlockType).caption}](${
+        (chunk as BlockType).link || ''
+      })`
+
+    case nodeTypes.listItemChild:
+      const isOL = chunk && chunk.parentType === nodeTypes.ol_list
+      const listIndex = 'listIndex' in chunk ? chunk.listIndex : undefined
+      const treatAsLeaf =
+        (chunk as BlockType).children.length === 1 &&
+        isLeafNode((chunk as BlockType).children[0])
+
+      let spacer = ''
+      for (let k = 0; listDepth > k; k++) {
+        if (isOL) {
+          // https://github.com/remarkjs/remark-react/issues/65
+          spacer += '   '
+        } else {
+          spacer += '  '
+        }
+      }
+      return `${spacer}${isOL ? `${listIndex}.` : '-'} ${children}${
+        treatAsLeaf ? '\n' : ''
+      }`
+
+    case nodeTypes.paragraph:
+      return `${children}\n`
+
+    case nodeTypes.thematic_break:
+      return `---\n`
+
+    default: {
+      if (opts.flavour === 'whatsapp') {
+        return children
+      }
+      return escapeHtml(children)
+    }
+  }
+}
+
+// This function handles the case of a string like this: "   foo   "
+// Where it would be invalid markdown to generate this: "**   foo   **"
+// We instead, want to trim the whitespace out, apply formatting, and then
+// bring the whitespace back. So our returned string looks like this: "   **foo**   "
+function retainWhitespaceAndFormat(string: string, format: string) {
+  // we keep this for a comparison later
+  const frozenString = string.trim()
+
+  // children will be mutated
+  let children = frozenString
+
+  // We reverse the right side formatting, to properly handle bold/italic and strikeThrough
+  // formats, so we can create ~~***FooBar***~~
+  const fullFormat = `${format}${children}${reverseStr(format)}`
+
+  // This conditions accounts for no whitespace in our string
+  // if we don't have any, we can return early.
+  if (children.length === string.length) {
+    return fullFormat
+  }
+
+  // if we do have whitespace, let's add our formatting around our trimmed string
+  // We reverse the right side formatting, to properly handle bold/italic and strikeThrough
+  // formats, so we can create ~~***FooBar***~~
+  const formattedString = format + children + reverseStr(format)
+
+  // and replace the non-whitespace content of the string
+  return string.replace(frozenString, formattedString)
+}
+
+const reverseStr = (string: string) => string.split('').reverse().join('')

pnpm-lock.yaml

@@ -689,9 +689,6 @@ importers:
       qs:
         specifier: 6.11.2
         version: 6.11.2
-      remark-slate:
-        specifier: 1.8.6
-        version: 1.8.6
       stripe:
         specifier: 12.13.0
         version: 12.13.0
@@ -1320,6 +1317,9 @@ importers:
       '@udecode/plate-common':
         specifier: 21.1.5
         version: 21.1.5(@babel/core@7.22.9)(@types/react@18.2.15)(react-dom@18.2.0)(react@18.2.0)(scheduler@0.23.0)(slate-history@0.93.0)(slate-react@0.94.2)(slate@0.94.1)
+      escape-html:
+        specifier: ^1.0.3
+        version: 1.0.3
       google-auth-library:
         specifier: 8.9.0
         version: 8.9.0
@@ -1329,9 +1329,6 @@ importers:
       minio:
         specifier: 7.1.3
         version: 7.1.3
-      remark-slate:
-        specifier: 1.8.6
-        version: 1.8.6
       stripe:
         specifier: 12.13.0
         version: 12.13.0
@@ -1360,6 +1357,9 @@ importers:
       '@typebot.io/tsconfig':
         specifier: workspace:*
         version: link:../tsconfig
+      '@types/escape-html':
+        specifier: ^1.0.4
+        version: 1.0.4
       '@types/nodemailer':
         specifier: 6.4.8
         version: 6.4.8
@@ -7714,9 +7714,9 @@ packages:
       '@types/trusted-types': 2.0.4
     dev: true
 
-  /@types/escape-html@1.0.2:
+  /@types/escape-html@1.0.4:
-    resolution: {integrity: sha512-gaBLT8pdcexFztLSPRtriHeXY/Kn4907uOCZ4Q3lncFBkheAWOuNt53ypsF8szgxbEJ513UeBzcf4utN0EzEwA==}
+    resolution: {integrity: sha512-qZ72SFTgUAZ5a7Tj6kf2SHLetiH5S6f8G5frB2SPQ3EyF02kxdyBFf4Tz4banE3xCgGnKgWLt//a6VuYHKYJTg==}
-    dev: false
+    dev: true
 
   /@types/estree-jsx@1.0.3:
     resolution: {integrity: sha512-pvQ+TKeRHeiUGRhvYwRrQ/ISnohKkSJR14fT2yqyZ4e9K5vqc7hrtY2Y1Dw0ZwAzQ6DQsxsaCUuSIIi8v0Cq6w==}
@@ -18593,13 +18593,6 @@ packages:
       vfile: 6.0.1
     dev: false
 
-  /remark-slate@1.8.6:
-    resolution: {integrity: sha512-1Gmt5MGw25MRVP+0xTXqw9JQDWfRNWujD4YFCPg036a9DZYhn7mLFjM6jreHB+9hKa6RCMOm5thiXznAmdn8Ug==}
-    dependencies:
-      '@types/escape-html': 1.0.2
-      escape-html: 1.0.3
-    dev: false
-
   /remark-stringify@10.0.3:
     resolution: {integrity: sha512-koyOzCMYoUHudypbj4XpnAKFbkddRMYZHwghnxd7ue5210WzGw6kOBwauJTRUMq16jsovXx8dYNvSSWP89kZ3A==}
     dependencies: