2
0

🚸 Rewrite the markdown deserializer to improve br… (#1198)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Refactor**
- Updated markdown handling and serialization libraries for improved
performance and accuracy in text formatting.
- **New Features**
- Enhanced rich text and markdown conversion capabilities, providing
users with more reliable and seamless text formatting options.
- **Documentation**
- Added detailed documentation for markdown to rich text conversion and
vice versa, ensuring easier understanding and implementation for
developers.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Baptiste Arnaud
2024-01-31 08:03:13 +01:00
committed by GitHub
parent 26872e2bac
commit ff9c4726cc
22 changed files with 1017 additions and 478 deletions

View File

@ -2879,6 +2879,21 @@
"type": "string"
},
"required": true
},
{
"in": "query",
"name": "timeFilter",
"schema": {
"type": "string",
"enum": [
"today",
"last7Days",
"last30Days",
"yearToDate",
"allTime"
],
"default": "today"
}
}
],
"responses": {
@ -3186,6 +3201,21 @@
"type": "string"
},
"required": true
},
{
"in": "query",
"name": "timeFilter",
"schema": {
"type": "string",
"enum": [
"today",
"last7Days",
"last30Days",
"yearToDate",
"allTime"
],
"default": "today"
}
}
],
"responses": {

View File

@ -18,7 +18,6 @@
"@typebot.io/tsconfig": "workspace:*",
"@typebot.io/variables": "workspace:*",
"@udecode/plate-common": "21.1.5",
"@udecode/plate-serializer-md": "24.4.0",
"ai": "2.2.31",
"chrono-node": "2.7.0",
"date-fns": "2.30.0",

View File

@ -11,13 +11,10 @@ import {
getVariablesToParseInfoInText,
parseVariables,
} from '@typebot.io/variables/parseVariables'
import { TDescendant, createPlateEditor } from '@udecode/plate-common'
import {
createDeserializeMdPlugin,
deserializeMd,
} from '@udecode/plate-serializer-md'
import { TDescendant } from '@udecode/plate-common'
import { BubbleBlockType } from '@typebot.io/schemas/features/blocks/bubbles/constants'
import { defaultVideoBubbleContent } from '@typebot.io/schemas/features/blocks/bubbles/video/constants'
import { convertMarkdownToRichText } from '@typebot.io/lib/markdown/convertMarkdownToRichText'
type Params = {
version: 1 | 2
@ -207,28 +204,3 @@ const applyElementStyleToDescendants = (
),
}
})
const convertMarkdownToRichText = (text: string): TDescendant[] => {
const spacesBefore = text.match(/^[\s]+/)
const spacesAfter = text.match(/[\s]+$/)
const plugins = [createDeserializeMdPlugin()]
return [
...(spacesBefore?.at(0)
? [
{
type: 'p',
text: spacesBefore.at(0),
},
]
: []),
...deserializeMd(createPlateEditor({ plugins }) as unknown as any, text),
...(spacesAfter?.at(0)
? [
{
type: 'p',
text: spacesAfter.at(0),
},
]
: []),
]
}

View File

@ -5,7 +5,7 @@ import { BubbleBlockType } from '@typebot.io/schemas/features/blocks/bubbles/con
import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants'
import { defaultPictureChoiceOptions } from '@typebot.io/schemas/features/blocks/inputs/pictureChoice/constants'
import { defaultChoiceInputOptions } from '@typebot.io/schemas/features/blocks/inputs/choice/constants'
import { convertRichTextToMarkdown } from '@typebot.io/lib/serializer/convertRichTextToMarkdown'
import { convertRichTextToMarkdown } from '@typebot.io/lib/markdown/convertRichTextToMarkdown'
export const convertInputToWhatsAppMessages = (
input: NonNullable<ContinueChatResponse['input']>,

View File

@ -3,7 +3,7 @@ import { WhatsAppSendingMessage } from '@typebot.io/schemas/features/whatsapp'
import { isSvgSrc } from '@typebot.io/lib/utils'
import { BubbleBlockType } from '@typebot.io/schemas/features/blocks/bubbles/constants'
import { VideoBubbleContentType } from '@typebot.io/schemas/features/blocks/bubbles/video/constants'
import { convertRichTextToMarkdown } from '@typebot.io/lib/serializer/convertRichTextToMarkdown'
import { convertRichTextToMarkdown } from '@typebot.io/lib/markdown/convertRichTextToMarkdown'
const mp4HttpsUrlRegex = /^https:\/\/.*\.mp4$/

View File

@ -0,0 +1,69 @@
import {
createPlateEditor,
createPluginFactory,
getPluginOptions,
isUrl,
Value,
} from '@udecode/plate-common'
import markdown from 'remark-parse'
import { unified } from 'unified'
import { DeserializeMdPlugin } from './deserializer/types'
import { remarkPlugin } from './remark-slate/remarkPlugin'
import { RemarkPluginOptions } from './remark-slate/types'
import { remarkDefaultElementRules } from './remark-slate/remarkDefaultElementRules'
import { remarkDefaultTextRules } from './remark-slate/remarkDefaultTextRules'
import { deserialize } from './deserializer/deserialize'
export const convertMarkdownToRichText = <V extends Value>(data: string) => {
const plugins = [createDeserializeMdPlugin()]
const editor = createPlateEditor({ plugins }) as unknown as any
const { elementRules, textRules, indentList } = getPluginOptions<
DeserializeMdPlugin,
V
>(editor, KEY_DESERIALIZE_MD)
const tree: any = unified()
.use(markdown)
.use(remarkPlugin, {
editor,
elementRules,
textRules,
indentList,
} as unknown as RemarkPluginOptions<V>)
.processSync(data)
return tree.result
}
export const KEY_DESERIALIZE_MD = 'deserializeMd'
const createDeserializeMdPlugin = createPluginFactory<DeserializeMdPlugin>({
key: KEY_DESERIALIZE_MD,
then: (editor) => ({
editor: {
insertData: {
format: 'text/plain',
query: ({ data, dataTransfer }) => {
const htmlData = dataTransfer.getData('text/html')
if (htmlData) return false
const { files } = dataTransfer
if (
!files?.length && // if content is simply a URL pass through to not break LinkPlugin
isUrl(data)
) {
return false
}
return true
},
getFragment: ({ data }) => deserialize<Value>(editor, data),
},
},
}),
options: {
elementRules: remarkDefaultElementRules,
textRules: remarkDefaultTextRules,
indentList: false,
},
})

View File

@ -1,41 +1,36 @@
import { TElement } from '@udecode/plate-common'
import serialize from './serialize'
import { defaultNodeTypes } from './ast-types'
import serialize from './serializer/serialize'
import { defaultNodeTypes } from './serializer/ast-types'
export const convertRichTextToMarkdown = (
richText: TElement[],
options?: { flavour?: 'common' | 'whatsapp' }
) => {
let extraNewLinesCount = 0
const test = richText
.reduce<string[]>((acc, node) => {
if (node.type === 'variable') {
return [
...acc,
...node.children.map(
(child) =>
serialize(child, {
nodeTypes: defaultNodeTypes,
flavour: options?.flavour,
}) as string
),
...node.children.reduce<string[]>((acc, node) => {
const serializedElement = serialize(node, {
nodeTypes: defaultNodeTypes,
flavour: options?.flavour,
}) as string
if (!serializedElement || serializedElement === '<br>\n\n')
return [...acc, '\n']
return [...acc, serializedElement]
}, []),
]
}
const serializedElement = serialize(node, {
nodeTypes: defaultNodeTypes,
flavour: options?.flavour,
})
if (!serializedElement || serializedElement === '<br>\n\n') {
if (extraNewLinesCount > 0) {
return [...acc, '']
}
extraNewLinesCount++
return acc
}
extraNewLinesCount = 0
if (!serializedElement || serializedElement === '<br>\n\n')
return [...acc, '\n']
return [...acc, serializedElement]
}, [])
.join('\n')
.join('')
return test.endsWith('\n') ? test.slice(0, -1) : test
}

View File

@ -0,0 +1,30 @@
import { getPluginOptions, PlateEditor, Value } from '@udecode/plate-common'
import markdown from 'remark-parse'
import { unified } from 'unified'
import { DeserializeMdPlugin } from './types'
import { remarkPlugin } from '../remark-slate/remarkPlugin'
import { RemarkPluginOptions } from '../remark-slate/types'
import { KEY_DESERIALIZE_MD } from '../convertMarkdownToRichText'
export const deserialize = <V extends Value>(
editor: PlateEditor<V>,
data: string
) => {
const { elementRules, textRules, indentList } = getPluginOptions<
DeserializeMdPlugin,
V
>(editor, KEY_DESERIALIZE_MD)
const tree: any = unified()
.use(markdown)
.use(remarkPlugin, {
editor,
elementRules,
textRules,
indentList,
} as unknown as RemarkPluginOptions<V>)
.processSync(data)
return tree.result
}

View File

@ -0,0 +1,8 @@
import { Value } from '@udecode/plate-common'
import { RemarkElementRules, RemarkTextRules } from '../remark-slate/types'
export interface DeserializeMdPlugin<V extends Value = Value> {
elementRules?: RemarkElementRules<V>
textRules?: RemarkTextRules<V>
indentList?: boolean
}

View File

@ -0,0 +1,253 @@
import { ELEMENT_BLOCKQUOTE } from '@udecode/plate-block-quote'
import {
ELEMENT_CODE_BLOCK,
ELEMENT_CODE_LINE,
} from '@udecode/plate-code-block'
import {
getPluginType,
TDescendant,
TElement,
TText,
Value,
} from '@udecode/plate-common'
import {
ELEMENT_H1,
ELEMENT_H2,
ELEMENT_H3,
ELEMENT_H4,
ELEMENT_H5,
ELEMENT_H6,
} from '@udecode/plate-heading'
import { ELEMENT_HR } from '@udecode/plate-horizontal-rule'
import { ELEMENT_LINK } from '@udecode/plate-link'
import {
ELEMENT_LI,
ELEMENT_LIC,
ELEMENT_OL,
ELEMENT_UL,
} from '@udecode/plate-list'
import { ELEMENT_IMAGE } from '@udecode/plate-media'
import { ELEMENT_PARAGRAPH } from '@udecode/plate-paragraph'
import { remarkTransformElementChildren } from './remarkTransformElementChildren'
import { MdastNode, RemarkElementRules, RemarkPluginOptions } from './types'
export const remarkDefaultElementRules: RemarkElementRules<Value> = {
heading: {
transform: (node, lastLineNumber, options) => {
const headingType = {
1: ELEMENT_H1,
2: ELEMENT_H2,
3: ELEMENT_H3,
4: ELEMENT_H4,
5: ELEMENT_H5,
6: ELEMENT_H6,
}[node.depth ?? 1]
return [
...parseLineBreakNodes(node, lastLineNumber, options),
{
type: getPluginType(options.editor, headingType),
children: remarkTransformElementChildren(
node,
lastLineNumber,
options
),
},
]
},
},
list: {
transform: (node, lastLineNumber, options) => {
if (options.indentList) {
const listStyleType = node.ordered ? 'decimal' : 'disc'
const parseListItems = (
_node: MdastNode,
listItems: TElement[] = [],
indent = 1
) => {
_node.children!.forEach((listItem) => {
const [paragraph, ...subLists] = listItem.children!
listItems.push({
type: getPluginType(options.editor, ELEMENT_PARAGRAPH),
listStyleType,
indent,
children: remarkTransformElementChildren(
paragraph || '',
lastLineNumber,
options
),
})
subLists.forEach((subList) => {
parseListItems(subList, listItems, indent + 1)
})
})
return listItems
}
return [
...parseLineBreakNodes(node, lastLineNumber, options),
parseListItems(node),
]
} else {
return [
...parseLineBreakNodes(node, lastLineNumber, options),
{
type: getPluginType(
options.editor,
node.ordered ? ELEMENT_OL : ELEMENT_UL
),
children: remarkTransformElementChildren(
node,
lastLineNumber,
options
),
},
]
}
},
},
listItem: {
transform: (node, lastLineNumber, options) => ({
type: getPluginType(options.editor, ELEMENT_LI),
children: remarkTransformElementChildren(
node,
lastLineNumber,
options
).map(
(child) =>
({
...child,
type:
child.type === getPluginType(options.editor, ELEMENT_PARAGRAPH)
? getPluginType(options.editor, ELEMENT_LIC)
: child.type,
} as TDescendant)
),
}),
},
paragraph: {
transform: (node, lastLineNumber, options) => {
const children = remarkTransformElementChildren(
node,
lastLineNumber,
options
)
const paragraphType = getPluginType(options.editor, ELEMENT_PARAGRAPH)
const splitBlockTypes = new Set([
getPluginType(options.editor, ELEMENT_IMAGE),
])
const elements: TElement[] = []
let inlineNodes: TDescendant[] = []
const flushInlineNodes = () => {
if (inlineNodes.length > 0) {
elements.push({
type: paragraphType,
children: inlineNodes,
})
inlineNodes = []
}
}
children.forEach((child) => {
const { type } = child
if (type && splitBlockTypes.has(type as string)) {
flushInlineNodes()
elements.push(child as TElement)
} else {
inlineNodes.push(child)
}
})
flushInlineNodes()
return [
...parseLineBreakNodes(node, lastLineNumber, options),
...elements,
]
},
},
link: {
transform: (node, lastLineNumber, options) => [
...parseLineBreakNodes(node, lastLineNumber, options),
{
type: getPluginType(options.editor, ELEMENT_LINK),
url: node.url,
children: remarkTransformElementChildren(node, lastLineNumber, options),
},
],
},
image: {
transform: (node, lastLineNumber, options) => [
...parseLineBreakNodes(node, lastLineNumber, options),
{
type: getPluginType(options.editor, ELEMENT_IMAGE),
children: [{ text: '' } as TText],
url: node.url,
caption: [{ text: node.alt } as TText],
},
],
},
blockquote: {
transform: (node, lastLineNumber, options) => [
...parseLineBreakNodes(node, lastLineNumber, options),
{
type: getPluginType(options.editor, ELEMENT_BLOCKQUOTE),
children: node.children!.flatMap((paragraph) =>
remarkTransformElementChildren(paragraph, lastLineNumber, options)
),
},
],
},
code: {
transform: (node, lastLineNumber, options) => [
...parseLineBreakNodes(node, lastLineNumber, options),
{
type: getPluginType(options.editor, ELEMENT_CODE_BLOCK),
lang: node.lang ?? undefined,
children: (node.value || '').split('\n').map((line) => ({
type: getPluginType(options.editor, ELEMENT_CODE_LINE),
children: [{ text: line } as TText],
})),
},
],
},
thematicBreak: {
transform: (node, lastLineNumber, options) => [
...parseLineBreakNodes(node, lastLineNumber, options),
{
type: getPluginType(options.editor, ELEMENT_HR),
children: [{ text: '' } as TText],
},
],
},
}
const parseLineBreakNodes = (
node: MdastNode,
lastLineNumber: number,
options: RemarkPluginOptions<Value>
) => {
const lineBreaks = node.position.start.line - lastLineNumber
let lineBreakNodes = []
if (lineBreaks > 1)
lineBreakNodes.push(
...Array(lineBreaks - 1).fill({
type: getPluginType(options.editor, ELEMENT_PARAGRAPH),
children: [{ text: '' } as TText],
})
)
return lineBreakNodes
}

View File

@ -0,0 +1,12 @@
import { MARK_BOLD, MARK_CODE, MARK_ITALIC } from '@udecode/plate-basic-marks'
import { getPluginType, Value } from '@udecode/plate-common'
import { RemarkTextRules } from './types'
export const remarkDefaultTextRules: RemarkTextRules<Value> = {
text: {},
emphasis: { mark: ({ editor }) => getPluginType(editor, MARK_ITALIC) },
strong: { mark: ({ editor }) => getPluginType(editor, MARK_BOLD) },
inlineCode: { mark: ({ editor }) => getPluginType(editor, MARK_CODE) },
html: { transform: (text: string) => text.replaceAll('<br>', '\n') },
}

View File

@ -0,0 +1,18 @@
import { Value } from '@udecode/plate-common'
import { remarkTransformNode } from './remarkTransformNode'
import { MdastNode, RemarkPluginOptions } from './types'
export function remarkPlugin<V extends Value>(options: RemarkPluginOptions<V>) {
let lastLineNumber = 1
const compiler = (node: { children: Array<MdastNode> }) => {
return node.children.flatMap((child) => {
const parsedChild = remarkTransformNode(child, lastLineNumber, options)
lastLineNumber = child.position?.end.line || lastLineNumber
return parsedChild
})
}
// @ts-ignore
this.Compiler = compiler
}

View File

@ -0,0 +1,10 @@
import { MdastNodeType } from './types'
export const remarkTextTypes: MdastNodeType[] = [
'emphasis',
'strong',
'delete',
'inlineCode',
'html',
'text',
]

View File

@ -0,0 +1,17 @@
import { TElement, Value } from '@udecode/plate-common'
import { MdastNode, RemarkPluginOptions } from './types'
export const remarkTransformElement = <V extends Value>(
node: MdastNode,
lastLineNumber: number,
options: RemarkPluginOptions<V>
): TElement | TElement[] => {
const { elementRules } = options
const { type } = node
const elementRule = (elementRules as any)[type!]
if (!elementRule) return []
return elementRule.transform(node, lastLineNumber, options)
}

View File

@ -0,0 +1,17 @@
import { TDescendant, Value } from '@udecode/plate-common'
import { remarkTransformNode } from './remarkTransformNode'
import { MdastNode, RemarkPluginOptions } from './types'
export const remarkTransformElementChildren = <V extends Value>(
node: MdastNode,
lastLineNumber: number,
options: RemarkPluginOptions<V>
): TDescendant[] => {
const { children } = node
if (!children) return []
return children.flatMap((child) =>
remarkTransformNode(child, lastLineNumber, options)
)
}

View File

@ -0,0 +1,20 @@
import { TDescendant, Value } from '@udecode/plate-common'
import { remarkTextTypes } from './remarkTextTypes'
import { remarkTransformElement } from './remarkTransformElement'
import { remarkTransformText } from './remarkTransformText'
import { MdastNode, RemarkPluginOptions } from './types'
export const remarkTransformNode = <V extends Value>(
node: MdastNode,
lastLineNumber: number,
options: RemarkPluginOptions<V>
): TDescendant | TDescendant[] => {
const { type } = node
if (remarkTextTypes.includes(type!)) {
return remarkTransformText(node, options)
}
return remarkTransformElement(node, lastLineNumber, options)
}

View File

@ -0,0 +1,36 @@
import { TText, Value } from '@udecode/plate-common'
import { remarkDefaultTextRules } from './remarkDefaultTextRules'
import { MdastNode, RemarkPluginOptions } from './types'
export const remarkTransformText = <V extends Value>(
node: MdastNode,
options: RemarkPluginOptions<V>,
inheritedMarkProps: { [key: string]: boolean } = {}
): TText | TText[] => {
const { editor, textRules } = options
const { type, value, children } = node
const textRule = (textRules as any)[type!] || remarkDefaultTextRules.text
const { mark, transform = (text: string) => text } = textRule
const markProps = mark
? {
...inheritedMarkProps,
[mark({ editor })]: true,
}
: inheritedMarkProps
const childTextNodes =
children?.flatMap((child) =>
remarkTransformText(child, options, markProps)
) || []
const currentTextNodes =
value || childTextNodes.length === 0
? [{ text: transform(value || ''), ...markProps } as TText]
: []
return [...currentTextNodes, ...childTextNodes]
}

View File

@ -0,0 +1,67 @@
import { PlateEditor, TElement, Value } from '@udecode/plate-common'
export type MdastElementType =
| 'paragraph'
| 'heading'
| 'list'
| 'listItem'
| 'link'
| 'image'
| 'blockquote'
| 'code'
| 'thematicBreak'
export type MdastTextType =
| 'emphasis'
| 'strong'
| 'delete'
| 'inlineCode'
| 'html'
| 'text'
export type MdastNodeType = MdastElementType | MdastTextType
export interface MdastNode {
type?: MdastNodeType
ordered?: boolean
value?: string
text?: string
children?: Array<MdastNode>
depth?: 1 | 2 | 3 | 4 | 5 | 6
url?: string
alt?: string
lang?: string
// mdast metadata
position?: any
spread?: any
checked?: any
indent?: any
}
export type RemarkElementRule<V extends Value> = {
transform: (
node: MdastNode,
lastLineNumber: number,
options: RemarkPluginOptions<V>
) => TElement | TElement[]
}
export type RemarkElementRules<V extends Value> = {
[key in MdastElementType]?: RemarkElementRule<V>
}
export type RemarkTextRule<V extends Value> = {
mark?: (options: RemarkPluginOptions<V>) => string
transform?: (text: string) => string
}
export type RemarkTextRules<V extends Value> = {
[key in MdastTextType]?: RemarkTextRule<V>
}
export type RemarkPluginOptions<V extends Value> = {
editor: PlateEditor<V>
elementRules: RemarkElementRules<V>
textRules: RemarkTextRules<V>
indentList?: boolean
}

View File

@ -26,12 +26,23 @@
"dependencies": {
"@sentry/nextjs": "7.77.0",
"@trpc/server": "10.40.0",
"@udecode/plate-basic-marks": "21.1.5",
"@udecode/plate-block-quote": "30.1.2",
"@udecode/plate-code-block": "30.1.2",
"@udecode/plate-common": "21.1.5",
"@udecode/plate-heading": "30.1.2",
"@udecode/plate-horizontal-rule": "30.1.2",
"@udecode/plate-link": "21.2.0",
"@udecode/plate-list": "30.1.2",
"@udecode/plate-media": "30.1.2",
"@udecode/plate-paragraph": "30.1.2",
"escape-html": "^1.0.3",
"google-auth-library": "8.9.0",
"got": "12.6.0",
"minio": "7.1.3",
"remark-parse": "11.0.0",
"stripe": "12.13.0",
"unified": "11.0.4",
"zod": "3.22.4"
}
}

827
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff