From 244a29423b6d355f55c97bfc4a14ee7a1aa13633 Mon Sep 17 00:00:00 2001 From: Baptiste Arnaud Date: Mon, 8 Jan 2024 08:40:25 +0100 Subject: [PATCH] :zap: (whatsapp) Improve / fix markdown serializer Forked remark-slate code to create Typebot's own serializer Closes #1056 --- .../public/templates/basic-chat-gpt.json | 41 +-- .../public/templates/chat-gpt-personas.json | 64 ++--- packages/bot-engine/package.json | 9 +- .../whatsapp/convertInputToWhatsAppMessage.ts | 6 +- .../convertMessageToWhatsAppMessage.ts | 6 +- .../whatsapp/convertRichTextToWhatsAppText.ts | 9 - packages/lib/package.json | 7 +- packages/lib/serializer/ast-types.ts | 243 ++++++++++++++++ .../serializer/convertRichTextToMarkdown.ts | 41 +++ packages/lib/serializer/serialize.ts | 266 ++++++++++++++++++ pnpm-lock.yaml | 25 +- 11 files changed, 629 insertions(+), 88 deletions(-) delete mode 100644 packages/bot-engine/whatsapp/convertRichTextToWhatsAppText.ts create mode 100644 packages/lib/serializer/ast-types.ts create mode 100644 packages/lib/serializer/convertRichTextToMarkdown.ts create mode 100644 packages/lib/serializer/serialize.ts diff --git a/apps/builder/public/templates/basic-chat-gpt.json b/apps/builder/public/templates/basic-chat-gpt.json index 4d3af96bc..a6dc537ac 100644 --- a/apps/builder/public/templates/basic-chat-gpt.json +++ b/apps/builder/public/templates/basic-chat-gpt.json @@ -92,7 +92,6 @@ "dialogueVariableId": "vabkycu0qqff5d6ar2ama16pf" } ], - "credentialsId": "clpjnjrbt00051aliw6610w1z", "responseMapping": [ { "item": "Message content", @@ -101,6 +100,16 @@ ] } }, + { + "id": "myldn1l1nfdwwm8qvza71rwv", + "type": "text", + "content": { + "richText": [ + { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] } + ] + }, + "groupId": "a6ymhjwtkqwp8t127plz8qmk" + }, { "id": "yblc864bzipaqfja7b2o3oo0", "type": "Set variable", @@ -108,17 +117,8 @@ "variableId": "vabkycu0qqff5d6ar2ama16pf", "type": "Append value(s)", "item": "{{Assistant Message}}" - } - }, - { - "id": "myldn1l1nfdwwm8qvza71rwv", - "outgoingEdgeId": "y8ml9ljnsydol9b42fd9zdve", - "type": "text", - "content": { - "richText": [ - { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] } - ] - } + }, + "outgoingEdgeId": "at8takz56suqmaul5teazymb" } ] }, @@ -155,20 +155,23 @@ "from": { "blockId": "gphm5wy1md9cunwkdtbzg6nq" }, "to": { "groupId": "qfrz5nwm63g12dajsjxothb5" } }, - { - "id": "y8ml9ljnsydol9b42fd9zdve", - "from": { "blockId": "myldn1l1nfdwwm8qvza71rwv" }, - "to": { "groupId": "qfrz5nwm63g12dajsjxothb5" } - }, { "id": "fpj0xacppqd1s5slyljzhzc9", "from": { "blockId": "m4jadtknjb3za3gvxj1xdn1k" }, "to": { "groupId": "a6ymhjwtkqwp8t127plz8qmk" } }, { + "id": "pn7omb9mx5xxc4mzq028fcmq", "from": { "eventId": "ewnfbo0exlu7ihfu2lu2lusm" }, - "to": { "groupId": "t3tv4dm3khwmiotjle5jb65g" }, - "id": "pn7omb9mx5xxc4mzq028fcmq" + "to": { "groupId": "t3tv4dm3khwmiotjle5jb65g" } + }, + { + "from": { + "blockId": "yblc864bzipaqfja7b2o3oo0", + "groupId": "a6ymhjwtkqwp8t127plz8qmk" + }, + "to": { "groupId": "qfrz5nwm63g12dajsjxothb5" }, + "id": "at8takz56suqmaul5teazymb" } ], "variables": [ diff --git a/apps/builder/public/templates/chat-gpt-personas.json b/apps/builder/public/templates/chat-gpt-personas.json index 995a76e36..6431b2efa 100644 --- a/apps/builder/public/templates/chat-gpt-personas.json +++ b/apps/builder/public/templates/chat-gpt-personas.json @@ -74,7 +74,6 @@ "dialogueVariableId": "vu9adij5penetej2xz89htfe6" } ], - "credentialsId": "clpjnjrbt00051aliw6610w1z", "responseMapping": [ { "item": "Message content", @@ -83,15 +82,6 @@ ] } }, - { - "id": "kftq9x1wnrcefzc268ydmqkn", - "type": "Set variable", - "options": { - "variableId": "vu9adij5penetej2xz89htfe6", - "type": "Append value(s)", - "item": "{{Assistant Message}}" - } - }, { "id": "myldn1l1nfdwwm8qvza71rwv", "type": "text", @@ -99,6 +89,16 @@ "richText": [ { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] } ] + }, + "groupId": "dmg57mgick51p8l5pnyqtyf9" + }, + { + "id": "kftq9x1wnrcefzc268ydmqkn", + "type": "Set variable", + "options": { + "variableId": "vu9adij5penetej2xz89htfe6", + "type": "Append value(s)", + "item": "{{Assistant Message}}" } }, { @@ -139,7 +139,6 @@ "dialogueVariableId": "vu9adij5penetej2xz89htfe6" } ], - "credentialsId": "clpjnjrbt00051aliw6610w1z", "responseMapping": [ { "item": "Message content", @@ -148,15 +147,6 @@ ] } }, - { - "id": "dyfigmu4095x8p99qe461zbh", - "type": "Set variable", - "options": { - "variableId": "vu9adij5penetej2xz89htfe6", - "type": "Append value(s)", - "item": "{{Assistant Message}}" - } - }, { "id": "sei88rrjcmpgm3vhxjvkofyt", "type": "text", @@ -164,6 +154,16 @@ "richText": [ { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] } ] + }, + "groupId": "fj5z2nx488htv0843kq6qeyk" + }, + { + "id": "dyfigmu4095x8p99qe461zbh", + "type": "Set variable", + "options": { + "variableId": "vu9adij5penetej2xz89htfe6", + "type": "Append value(s)", + "item": "{{Assistant Message}}" } }, { @@ -204,7 +204,6 @@ "dialogueVariableId": "vu9adij5penetej2xz89htfe6" } ], - "credentialsId": "clpjnjrbt00051aliw6610w1z", "responseMapping": [ { "item": "Message content", @@ -213,15 +212,6 @@ ] } }, - { - "id": "usolgxcte60rin18ccygzbdu", - "type": "Set variable", - "options": { - "variableId": "vu9adij5penetej2xz89htfe6", - "type": "Append value(s)", - "item": "{{Assistant Message}}" - } - }, { "id": "h96lile0evtqa0jx24gmfo25", "type": "text", @@ -229,6 +219,16 @@ "richText": [ { "type": "p", "children": [{ "text": "{{Assistant Message}}" }] } ] + }, + "groupId": "csbysu8dr08zxr4i6hzvzjdf" + }, + { + "id": "usolgxcte60rin18ccygzbdu", + "type": "Set variable", + "options": { + "variableId": "vu9adij5penetej2xz89htfe6", + "type": "Append value(s)", + "item": "{{Assistant Message}}" } }, { @@ -678,9 +678,9 @@ "to": { "groupId": "dmg57mgick51p8l5pnyqtyf9" } }, { + "id": "k5bj58emklqfqv3hemko4u23", "from": { "eventId": "w99qhdr20tw02sfrfwkfc1tg" }, - "to": { "groupId": "bofjp88arodr4k0btv2esyqy" }, - "id": "k5bj58emklqfqv3hemko4u23" + "to": { "groupId": "bofjp88arodr4k0btv2esyqy" } } ], "variables": [ diff --git a/packages/bot-engine/package.json b/packages/bot-engine/package.json index a0a0fe9b2..55e85ca67 100644 --- a/packages/bot-engine/package.json +++ b/packages/bot-engine/package.json @@ -30,14 +30,13 @@ "nodemailer": "6.9.3", "openai": "4.19.0", "qs": "6.11.2", - "remark-slate": "1.8.6", "stripe": "12.13.0" }, "devDependencies": { - "@types/nodemailer": "6.4.8", - "@types/qs": "6.9.7", - "@typebot.io/forge-schemas": "workspace:*", "@typebot.io/forge": "workspace:*", - "@typebot.io/forge-repository": "workspace:*" + "@typebot.io/forge-repository": "workspace:*", + "@typebot.io/forge-schemas": "workspace:*", + "@types/nodemailer": "6.4.8", + "@types/qs": "6.9.7" } } diff --git a/packages/bot-engine/whatsapp/convertInputToWhatsAppMessage.ts b/packages/bot-engine/whatsapp/convertInputToWhatsAppMessage.ts index 524cb0ef8..efdfe5549 100644 --- a/packages/bot-engine/whatsapp/convertInputToWhatsAppMessage.ts +++ b/packages/bot-engine/whatsapp/convertInputToWhatsAppMessage.ts @@ -1,11 +1,11 @@ import { ButtonItem, ContinueChatResponse } from '@typebot.io/schemas' import { WhatsAppSendingMessage } from '@typebot.io/schemas/features/whatsapp' -import { convertRichTextToWhatsAppText } from './convertRichTextToWhatsAppText' import { isDefined, isEmpty } from '@typebot.io/lib/utils' import { BubbleBlockType } from '@typebot.io/schemas/features/blocks/bubbles/constants' import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants' import { defaultPictureChoiceOptions } from '@typebot.io/schemas/features/blocks/inputs/pictureChoice/constants' import { defaultChoiceInputOptions } from '@typebot.io/schemas/features/blocks/inputs/choice/constants' +import { convertRichTextToMarkdown } from '@typebot.io/lib/serializer/convertRichTextToMarkdown' export const convertInputToWhatsAppMessages = ( input: NonNullable, @@ -13,7 +13,9 @@ export const convertInputToWhatsAppMessages = ( ): WhatsAppSendingMessage[] => { const lastMessageText = lastMessage?.type === BubbleBlockType.TEXT - ? convertRichTextToWhatsAppText(lastMessage.content.richText ?? []) + ? convertRichTextToMarkdown(lastMessage.content.richText ?? [], { + flavour: 'whatsapp', + }) : undefined switch (input.type) { case InputBlockType.DATE: diff --git a/packages/bot-engine/whatsapp/convertMessageToWhatsAppMessage.ts b/packages/bot-engine/whatsapp/convertMessageToWhatsAppMessage.ts index 837e1d618..d31ae92e2 100644 --- a/packages/bot-engine/whatsapp/convertMessageToWhatsAppMessage.ts +++ b/packages/bot-engine/whatsapp/convertMessageToWhatsAppMessage.ts @@ -1,9 +1,9 @@ import { ContinueChatResponse } from '@typebot.io/schemas' import { WhatsAppSendingMessage } from '@typebot.io/schemas/features/whatsapp' -import { convertRichTextToWhatsAppText } from './convertRichTextToWhatsAppText' import { isSvgSrc } from '@typebot.io/lib/utils' import { BubbleBlockType } from '@typebot.io/schemas/features/blocks/bubbles/constants' import { VideoBubbleContentType } from '@typebot.io/schemas/features/blocks/bubbles/video/constants' +import { convertRichTextToMarkdown } from '@typebot.io/lib/serializer/convertRichTextToMarkdown' const mp4HttpsUrlRegex = /^https:\/\/.*\.mp4$/ @@ -17,7 +17,9 @@ export const convertMessageToWhatsAppMessage = ( return { type: 'text', text: { - body: convertRichTextToWhatsAppText(message.content.richText), + body: convertRichTextToMarkdown(message.content.richText, { + flavour: 'whatsapp', + }), }, } } diff --git a/packages/bot-engine/whatsapp/convertRichTextToWhatsAppText.ts b/packages/bot-engine/whatsapp/convertRichTextToWhatsAppText.ts deleted file mode 100644 index 22a3015c0..000000000 --- a/packages/bot-engine/whatsapp/convertRichTextToWhatsAppText.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { TElement } from '@udecode/plate-common' -import { serialize } from 'remark-slate' - -export const convertRichTextToWhatsAppText = (richText: TElement[]): string => - richText - .map((chunk) => - serialize(chunk)?.replaceAll('**', '*').replaceAll(''', "'") - ) - .join('\n') diff --git a/packages/lib/package.json b/packages/lib/package.json index 2c2dd284c..ac30cdad4 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -9,14 +9,15 @@ "@paralleldrive/cuid2": "2.2.1", "@playwright/test": "1.36.0", "@typebot.io/env": "workspace:*", + "@typebot.io/forge-repository": "workspace:*", "@typebot.io/prisma": "workspace:*", "@typebot.io/schemas": "workspace:*", "@typebot.io/tsconfig": "workspace:*", + "@types/escape-html": "^1.0.4", "@types/nodemailer": "6.4.8", "next": "14.0.3", "nodemailer": "6.9.3", - "typescript": "5.3.2", - "@typebot.io/forge-repository": "workspace:*" + "typescript": "5.3.2" }, "peerDependencies": { "next": "14.0.0", @@ -26,10 +27,10 @@ "@sentry/nextjs": "7.77.0", "@trpc/server": "10.40.0", "@udecode/plate-common": "21.1.5", + "escape-html": "^1.0.3", "google-auth-library": "8.9.0", "got": "12.6.0", "minio": "7.1.3", - "remark-slate": "1.8.6", "stripe": "12.13.0", "zod": "3.22.4" } diff --git a/packages/lib/serializer/ast-types.ts b/packages/lib/serializer/ast-types.ts new file mode 100644 index 000000000..daacebc8a --- /dev/null +++ b/packages/lib/serializer/ast-types.ts @@ -0,0 +1,243 @@ +export interface NodeTypes { + paragraph: string + blockquote: string + code_block: string + link: string + ul_list: string + ol_list: string + listItem: string + listItemChild: string + heading: { + 1: string + 2: string + 3: string + 4: string + 5: string + 6: string + } + emphasis_mark: string + strong_mark: string + delete_mark: string + inline_code_mark: string + thematic_break: string + image: string + variable: string + ['inline-variable']: string +} + +export type MdastNodeType = + | 'paragraph' + | 'heading' + | 'list' + | 'listItem' + | 'link' + | 'image' + | 'blockquote' + | 'code' + | 'html' + | 'emphasis' + | 'strong' + | 'delete' + | 'inlineCode' + | 'thematicBreak' + | 'text' + +export const defaultNodeTypes: NodeTypes = { + paragraph: 'p', + blockquote: 'blockquote', + code_block: 'code_block', + link: 'a', + ul_list: 'ul', + ol_list: 'ol', + listItem: 'li', + listItemChild: 'lic', + heading: { + 1: 'h1', + 2: 'h2', + 3: 'h3', + 4: 'h4', + 5: 'h5', + 6: 'h6', + }, + emphasis_mark: 'italic', + strong_mark: 'bold', + delete_mark: 'strikeThrough', + inline_code_mark: 'code', + thematic_break: 'thematic_break', + image: 'image', + variable: 'variable', + ['inline-variable']: 'inline-variable', +} + +export interface LeafType { + text: string + strikeThrough?: boolean + bold?: boolean + italic?: boolean + code?: boolean + parentType?: string +} + +export interface BlockType { + type: string + parentType?: string + link?: string + caption?: string + language?: string + break?: boolean + listIndex?: number + children: Array +} + +export interface InputNodeTypes { + paragraph: string + block_quote: string + code_block: string + link: string + ul_list: string + ol_list: string + listItem: string + heading: { + 1: string + 2: string + 3: string + 4: string + 5: string + 6: string + } + emphasis_mark: string + strong_mark: string + delete_mark: string + inline_code_mark: string + thematic_break: string + image: string +} + +type RecursivePartial = { + [P in keyof T]?: RecursivePartial +} + +export interface OptionType { + nodeTypes?: RecursivePartial + linkDestinationKey?: string + imageSourceKey?: string + imageCaptionKey?: string +} + +export interface MdastNode { + type?: MdastNodeType + ordered?: boolean + value?: string + text?: string + children?: Array + depth?: 1 | 2 | 3 | 4 | 5 | 6 + url?: string + alt?: string + lang?: string + // mdast metadata + position?: any + spread?: any + checked?: any + indent?: any +} + +export type TextNode = { text?: string | undefined } + +export type CodeBlockNode = { + type: T['code_block'] + language: string | undefined + children: Array +} + +export type HeadingNode = { + type: + | T['heading'][1] + | T['heading'][2] + | T['heading'][3] + | T['heading'][4] + | T['heading'][5] + | T['heading'][6] + children: Array> +} + +export type ListNode = { + type: T['ol_list'] | T['ul_list'] + children: Array> +} + +export type ListItemNode = { + type: T['listItem'] + children: Array> +} + +export type ParagraphNode = { + type: T['paragraph'] + break?: true + children: Array> +} + +export type LinkNode = { + type: T['link'] + children: Array> + [urlKey: string]: string | undefined | Array> +} + +export type ImageNode = { + type: T['image'] + children: Array> + [sourceOrCaptionKey: string]: string | undefined | Array> +} + +export type BlockQuoteNode = { + type: T['block_quote'] + children: Array> +} + +export type InlineCodeMarkNode = { + type: T['inline_code_mark'] + children: Array + language: string | undefined +} + +export type ThematicBreakNode = { + type: T['thematic_break'] + children: Array> +} + +export type ItalicNode = { + [K in T['emphasis_mark']]: true +} & { + children: TextNode +} + +export type BoldNode = { + bold: true + children: TextNode +} + +export type StrikeThoughNode = { + strikeThrough: true + children: TextNode +} + +export type InlineCodeNode = { + code: true + text: string | undefined +} + +export type DeserializedNode = + | CodeBlockNode + | HeadingNode + | ListNode + | ListItemNode + | ParagraphNode + | LinkNode + | ImageNode + | BlockQuoteNode + | InlineCodeMarkNode + | ThematicBreakNode + | ItalicNode + | BoldNode + | StrikeThoughNode + | InlineCodeNode + | TextNode diff --git a/packages/lib/serializer/convertRichTextToMarkdown.ts b/packages/lib/serializer/convertRichTextToMarkdown.ts new file mode 100644 index 000000000..65fb4b776 --- /dev/null +++ b/packages/lib/serializer/convertRichTextToMarkdown.ts @@ -0,0 +1,41 @@ +import { TElement } from '@udecode/plate-common' +import serialize from './serialize' +import { defaultNodeTypes } from './ast-types' + +export const convertRichTextToMarkdown = ( + richText: TElement[], + options?: { flavour?: 'common' | 'whatsapp' } +) => { + let extraNewLinesCount = 0 + const test = richText + .reduce((acc, node) => { + if (node.type === 'variable' || node.type === 'inline-variable') { + return [ + ...acc, + ...node.children.map( + (child) => + serialize(child, { + nodeTypes: defaultNodeTypes, + flavour: options?.flavour, + }) as string + ), + ] + } + const serializedElement = serialize(node, { + nodeTypes: defaultNodeTypes, + flavour: options?.flavour, + }) + if (!serializedElement || serializedElement === '
\n\n') { + if (extraNewLinesCount > 0) { + return [...acc, ''] + } + extraNewLinesCount++ + return acc + } + extraNewLinesCount = 0 + return [...acc, serializedElement] + }, []) + .join('\n') + + return test.endsWith('\n') ? test.slice(0, -2) : test +} diff --git a/packages/lib/serializer/serialize.ts b/packages/lib/serializer/serialize.ts new file mode 100644 index 000000000..fc4482f80 --- /dev/null +++ b/packages/lib/serializer/serialize.ts @@ -0,0 +1,266 @@ +import { BlockType, defaultNodeTypes, LeafType, NodeTypes } from './ast-types' +import escapeHtml from 'escape-html' + +interface Options { + nodeTypes: NodeTypes + flavour?: 'common' | 'whatsapp' + listDepth?: number + ignoreParagraphNewline?: boolean +} + +const isLeafNode = (node: BlockType | LeafType): node is LeafType => { + return typeof (node as LeafType).text === 'string' +} + +const VOID_ELEMENTS: Array = ['thematic_break', 'image'] + +const BREAK_TAG = '
' + +export default function serialize( + chunk: BlockType | LeafType, + opts: Options = { + nodeTypes: defaultNodeTypes, + } +) { + const { + nodeTypes: userNodeTypes = defaultNodeTypes, + ignoreParagraphNewline = false, + listDepth = -1, + } = opts + + let text = (chunk as LeafType).text || '' + let type = (chunk as BlockType).type || '' + + const nodeTypes: NodeTypes = { + ...defaultNodeTypes, + ...userNodeTypes, + heading: { + ...defaultNodeTypes.heading, + ...userNodeTypes.heading, + }, + } + + const LIST_TYPES = [nodeTypes.ul_list, nodeTypes.ol_list] + + let children = text + + if (!isLeafNode(chunk)) { + children = chunk.children + .map((c: BlockType | LeafType, index) => { + const isList = !isLeafNode(c) + ? (LIST_TYPES as string[]).includes(c.type || '') + : false + + const selfIsList = (LIST_TYPES as string[]).includes(chunk.type || '') + + // Links can have the following shape + // In which case we don't want to surround + // with break tags + // { + // type: 'paragraph', + // children: [ + // { text: '' }, + // { type: 'link', children: [{ text: foo.com }]} + // { text: '' } + // ] + // } + let childrenHasLink = false + + if (!isLeafNode(chunk) && Array.isArray(chunk.children)) { + childrenHasLink = chunk.children.some( + (f) => !isLeafNode(f) && f.type === nodeTypes.link + ) + } + + return serialize( + { + ...c, + parentType: type === nodeTypes.listItem ? chunk.parentType : type, + listIndex: + type === nodeTypes.listItem ? chunk.listIndex : index + 1, + }, + { + flavour: opts.flavour, + nodeTypes, + // WOAH. + // what we're doing here is pretty tricky, it relates to the block below where + // we check for ignoreParagraphNewline and set type to paragraph. + // We want to strip out empty paragraphs sometimes, but other times we don't. + // If we're the descendant of a list, we know we don't want a bunch + // of whitespace. If we're parallel to a link we also don't want + // to respect neighboring paragraphs + ignoreParagraphNewline: + (ignoreParagraphNewline || + isList || + selfIsList || + childrenHasLink) && + // if we have c.break, never ignore empty paragraph new line + !(c as BlockType).break, + + // track depth of nested lists so we can add proper spacing + listDepth: (LIST_TYPES as string[]).includes( + (c as BlockType).type || '' + ) + ? listDepth + 1 + : listDepth, + } + ) + }) + .join('') + } + + // This is pretty fragile code, check the long comment where we iterate over children + if ( + !ignoreParagraphNewline && + (text === '' || text === '\n') && + chunk.parentType === nodeTypes.paragraph + ) { + type = nodeTypes.paragraph + children = BREAK_TAG + } + + if (children === '' && !VOID_ELEMENTS.find((k) => nodeTypes[k] === type)) + return + + // Never allow decorating break tags with rich text formatting, + // this can malform generated markdown + // Also ensure we're only ever applying text formatting to leaf node + // level chunks, otherwise we can end up in a situation where + // we try applying formatting like to a node like this: + // "Text foo bar **baz**" resulting in "**Text foo bar **baz****" + // which is invalid markup and can mess everything up + if (children !== BREAK_TAG && isLeafNode(chunk)) { + if (chunk.strikeThrough && chunk.bold && chunk.italic) { + if (opts.flavour === 'whatsapp') { + children = retainWhitespaceAndFormat(children, '*_~') + } else { + children = retainWhitespaceAndFormat(children, '~~***') + } + } else if (chunk.bold && chunk.italic) { + if (opts.flavour === 'whatsapp') { + children = retainWhitespaceAndFormat(children, '*_') + } else { + children = retainWhitespaceAndFormat(children, '***') + } + } else { + if (chunk.bold) { + if (opts.flavour === 'whatsapp') { + children = retainWhitespaceAndFormat(children, '*') + } else { + children = retainWhitespaceAndFormat(children, '**') + } + } + + if (chunk.italic) { + children = retainWhitespaceAndFormat(children, '_') + } + + if (chunk.strikeThrough) { + children = retainWhitespaceAndFormat(children, '~~') + } + + if (chunk.code) { + if (opts.flavour === 'whatsapp') { + children = retainWhitespaceAndFormat(children, '```') + } else { + children = retainWhitespaceAndFormat(children, '`') + } + } + } + } + + switch (type) { + case nodeTypes.heading[1]: + return `# ${children}\n` + case nodeTypes.heading[2]: + return `## ${children}\n` + case nodeTypes.heading[3]: + return `### ${children}\n` + case nodeTypes.heading[4]: + return `#### ${children}\n` + case nodeTypes.heading[5]: + return `##### ${children}\n` + case nodeTypes.heading[6]: + return `###### ${children}\n` + + case nodeTypes.blockquote: + return `> ${children}\n` + + case nodeTypes.code_block: + return `\`\`\`${ + (chunk as BlockType).language || '' + }\n${children}\n\`\`\`\n` + + case nodeTypes.link: + return `[${children}](${(chunk as any).url || ''})` + case nodeTypes.image: + return `![${(chunk as BlockType).caption}](${ + (chunk as BlockType).link || '' + })` + + case nodeTypes.listItemChild: + const isOL = chunk && chunk.parentType === nodeTypes.ol_list + const listIndex = 'listIndex' in chunk ? chunk.listIndex : undefined + const treatAsLeaf = + (chunk as BlockType).children.length === 1 && + isLeafNode((chunk as BlockType).children[0]) + + let spacer = '' + for (let k = 0; listDepth > k; k++) { + if (isOL) { + // https://github.com/remarkjs/remark-react/issues/65 + spacer += ' ' + } else { + spacer += ' ' + } + } + return `${spacer}${isOL ? `${listIndex}.` : '-'} ${children}${ + treatAsLeaf ? '\n' : '' + }` + + case nodeTypes.paragraph: + return `${children}\n` + + case nodeTypes.thematic_break: + return `---\n` + + default: { + if (opts.flavour === 'whatsapp') { + return children + } + return escapeHtml(children) + } + } +} + +// This function handles the case of a string like this: " foo " +// Where it would be invalid markdown to generate this: "** foo **" +// We instead, want to trim the whitespace out, apply formatting, and then +// bring the whitespace back. So our returned string looks like this: " **foo** " +function retainWhitespaceAndFormat(string: string, format: string) { + // we keep this for a comparison later + const frozenString = string.trim() + + // children will be mutated + let children = frozenString + + // We reverse the right side formatting, to properly handle bold/italic and strikeThrough + // formats, so we can create ~~***FooBar***~~ + const fullFormat = `${format}${children}${reverseStr(format)}` + + // This conditions accounts for no whitespace in our string + // if we don't have any, we can return early. + if (children.length === string.length) { + return fullFormat + } + + // if we do have whitespace, let's add our formatting around our trimmed string + // We reverse the right side formatting, to properly handle bold/italic and strikeThrough + // formats, so we can create ~~***FooBar***~~ + const formattedString = format + children + reverseStr(format) + + // and replace the non-whitespace content of the string + return string.replace(frozenString, formattedString) +} + +const reverseStr = (string: string) => string.split('').reverse().join('') diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c22175198..533800c6a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -689,9 +689,6 @@ importers: qs: specifier: 6.11.2 version: 6.11.2 - remark-slate: - specifier: 1.8.6 - version: 1.8.6 stripe: specifier: 12.13.0 version: 12.13.0 @@ -1320,6 +1317,9 @@ importers: '@udecode/plate-common': specifier: 21.1.5 version: 21.1.5(@babel/core@7.22.9)(@types/react@18.2.15)(react-dom@18.2.0)(react@18.2.0)(scheduler@0.23.0)(slate-history@0.93.0)(slate-react@0.94.2)(slate@0.94.1) + escape-html: + specifier: ^1.0.3 + version: 1.0.3 google-auth-library: specifier: 8.9.0 version: 8.9.0 @@ -1329,9 +1329,6 @@ importers: minio: specifier: 7.1.3 version: 7.1.3 - remark-slate: - specifier: 1.8.6 - version: 1.8.6 stripe: specifier: 12.13.0 version: 12.13.0 @@ -1360,6 +1357,9 @@ importers: '@typebot.io/tsconfig': specifier: workspace:* version: link:../tsconfig + '@types/escape-html': + specifier: ^1.0.4 + version: 1.0.4 '@types/nodemailer': specifier: 6.4.8 version: 6.4.8 @@ -7714,9 +7714,9 @@ packages: '@types/trusted-types': 2.0.4 dev: true - /@types/escape-html@1.0.2: - resolution: {integrity: sha512-gaBLT8pdcexFztLSPRtriHeXY/Kn4907uOCZ4Q3lncFBkheAWOuNt53ypsF8szgxbEJ513UeBzcf4utN0EzEwA==} - dev: false + /@types/escape-html@1.0.4: + resolution: {integrity: sha512-qZ72SFTgUAZ5a7Tj6kf2SHLetiH5S6f8G5frB2SPQ3EyF02kxdyBFf4Tz4banE3xCgGnKgWLt//a6VuYHKYJTg==} + dev: true /@types/estree-jsx@1.0.3: resolution: {integrity: sha512-pvQ+TKeRHeiUGRhvYwRrQ/ISnohKkSJR14fT2yqyZ4e9K5vqc7hrtY2Y1Dw0ZwAzQ6DQsxsaCUuSIIi8v0Cq6w==} @@ -18593,13 +18593,6 @@ packages: vfile: 6.0.1 dev: false - /remark-slate@1.8.6: - resolution: {integrity: sha512-1Gmt5MGw25MRVP+0xTXqw9JQDWfRNWujD4YFCPg036a9DZYhn7mLFjM6jreHB+9hKa6RCMOm5thiXznAmdn8Ug==} - dependencies: - '@types/escape-html': 1.0.2 - escape-html: 1.0.3 - dev: false - /remark-stringify@10.0.3: resolution: {integrity: sha512-koyOzCMYoUHudypbj4XpnAKFbkddRMYZHwghnxd7ue5210WzGw6kOBwauJTRUMq16jsovXx8dYNvSSWP89kZ3A==} dependencies: