🚸 Rewrite the markdown deserializer to improve br… (#1198)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Refactor**
- Updated markdown handling and serialization libraries for improved
performance and accuracy in text formatting.
- **New Features**
- Enhanced rich text and markdown conversion capabilities, providing
users with more reliable and seamless text formatting options.
- **Documentation**
- Added detailed documentation for markdown to rich text conversion and
vice versa, ensuring easier understanding and implementation for
developers.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/lib/markdown/convertMarkdownToRichText.ts

@@ -0,0 +1,69 @@
+import {
+  createPlateEditor,
+  createPluginFactory,
+  getPluginOptions,
+  isUrl,
+  Value,
+} from '@udecode/plate-common'
+import markdown from 'remark-parse'
+import { unified } from 'unified'
+
+import { DeserializeMdPlugin } from './deserializer/types'
+import { remarkPlugin } from './remark-slate/remarkPlugin'
+import { RemarkPluginOptions } from './remark-slate/types'
+import { remarkDefaultElementRules } from './remark-slate/remarkDefaultElementRules'
+import { remarkDefaultTextRules } from './remark-slate/remarkDefaultTextRules'
+import { deserialize } from './deserializer/deserialize'
+
+export const convertMarkdownToRichText = <V extends Value>(data: string) => {
+  const plugins = [createDeserializeMdPlugin()]
+  const editor = createPlateEditor({ plugins }) as unknown as any
+  const { elementRules, textRules, indentList } = getPluginOptions<
+    DeserializeMdPlugin,
+    V
+  >(editor, KEY_DESERIALIZE_MD)
+
+  const tree: any = unified()
+    .use(markdown)
+    .use(remarkPlugin, {
+      editor,
+      elementRules,
+      textRules,
+      indentList,
+    } as unknown as RemarkPluginOptions<V>)
+    .processSync(data)
+
+  return tree.result
+}
+
+export const KEY_DESERIALIZE_MD = 'deserializeMd'
+
+const createDeserializeMdPlugin = createPluginFactory<DeserializeMdPlugin>({
+  key: KEY_DESERIALIZE_MD,
+  then: (editor) => ({
+    editor: {
+      insertData: {
+        format: 'text/plain',
+        query: ({ data, dataTransfer }) => {
+          const htmlData = dataTransfer.getData('text/html')
+          if (htmlData) return false
+
+          const { files } = dataTransfer
+          if (
+            !files?.length && // if content is simply a URL pass through to not break LinkPlugin
+            isUrl(data)
+          ) {
+            return false
+          }
+          return true
+        },
+        getFragment: ({ data }) => deserialize<Value>(editor, data),
+      },
+    },
+  }),
+  options: {
+    elementRules: remarkDefaultElementRules,
+    textRules: remarkDefaultTextRules,
+    indentList: false,
+  },
+})

packages/lib/markdown/convertRichTextToMarkdown.ts

@@ -0,0 +1,36 @@
+import { TElement } from '@udecode/plate-common'
+import serialize from './serializer/serialize'
+import { defaultNodeTypes } from './serializer/ast-types'
+
+export const convertRichTextToMarkdown = (
+  richText: TElement[],
+  options?: { flavour?: 'common' | 'whatsapp' }
+) => {
+  const test = richText
+    .reduce<string[]>((acc, node) => {
+      if (node.type === 'variable') {
+        return [
+          ...acc,
+          ...node.children.reduce<string[]>((acc, node) => {
+            const serializedElement = serialize(node, {
+              nodeTypes: defaultNodeTypes,
+              flavour: options?.flavour,
+            }) as string
+            if (!serializedElement || serializedElement === '<br>\n\n')
+              return [...acc, '\n']
+            return [...acc, serializedElement]
+          }, []),
+        ]
+      }
+      const serializedElement = serialize(node, {
+        nodeTypes: defaultNodeTypes,
+        flavour: options?.flavour,
+      })
+      if (!serializedElement || serializedElement === '<br>\n\n')
+        return [...acc, '\n']
+      return [...acc, serializedElement]
+    }, [])
+    .join('')
+
+  return test.endsWith('\n') ? test.slice(0, -1) : test
+}

packages/lib/markdown/deserializer/deserialize.ts

@@ -0,0 +1,30 @@
+import { getPluginOptions, PlateEditor, Value } from '@udecode/plate-common'
+import markdown from 'remark-parse'
+import { unified } from 'unified'
+
+import { DeserializeMdPlugin } from './types'
+import { remarkPlugin } from '../remark-slate/remarkPlugin'
+import { RemarkPluginOptions } from '../remark-slate/types'
+import { KEY_DESERIALIZE_MD } from '../convertMarkdownToRichText'
+
+export const deserialize = <V extends Value>(
+  editor: PlateEditor<V>,
+  data: string
+) => {
+  const { elementRules, textRules, indentList } = getPluginOptions<
+    DeserializeMdPlugin,
+    V
+  >(editor, KEY_DESERIALIZE_MD)
+
+  const tree: any = unified()
+    .use(markdown)
+    .use(remarkPlugin, {
+      editor,
+      elementRules,
+      textRules,
+      indentList,
+    } as unknown as RemarkPluginOptions<V>)
+    .processSync(data)
+
+  return tree.result
+}

packages/lib/markdown/deserializer/types.ts

@@ -0,0 +1,8 @@
+import { Value } from '@udecode/plate-common'
+import { RemarkElementRules, RemarkTextRules } from '../remark-slate/types'
+
+export interface DeserializeMdPlugin<V extends Value = Value> {
+  elementRules?: RemarkElementRules<V>
+  textRules?: RemarkTextRules<V>
+  indentList?: boolean
+}

packages/lib/markdown/remark-slate/remarkDefaultElementRules.ts

@@ -0,0 +1,253 @@
+import { ELEMENT_BLOCKQUOTE } from '@udecode/plate-block-quote'
+import {
+  ELEMENT_CODE_BLOCK,
+  ELEMENT_CODE_LINE,
+} from '@udecode/plate-code-block'
+import {
+  getPluginType,
+  TDescendant,
+  TElement,
+  TText,
+  Value,
+} from '@udecode/plate-common'
+import {
+  ELEMENT_H1,
+  ELEMENT_H2,
+  ELEMENT_H3,
+  ELEMENT_H4,
+  ELEMENT_H5,
+  ELEMENT_H6,
+} from '@udecode/plate-heading'
+import { ELEMENT_HR } from '@udecode/plate-horizontal-rule'
+import { ELEMENT_LINK } from '@udecode/plate-link'
+import {
+  ELEMENT_LI,
+  ELEMENT_LIC,
+  ELEMENT_OL,
+  ELEMENT_UL,
+} from '@udecode/plate-list'
+import { ELEMENT_IMAGE } from '@udecode/plate-media'
+import { ELEMENT_PARAGRAPH } from '@udecode/plate-paragraph'
+
+import { remarkTransformElementChildren } from './remarkTransformElementChildren'
+import { MdastNode, RemarkElementRules, RemarkPluginOptions } from './types'
+
+export const remarkDefaultElementRules: RemarkElementRules<Value> = {
+  heading: {
+    transform: (node, lastLineNumber, options) => {
+      const headingType = {
+        1: ELEMENT_H1,
+        2: ELEMENT_H2,
+        3: ELEMENT_H3,
+        4: ELEMENT_H4,
+        5: ELEMENT_H5,
+        6: ELEMENT_H6,
+      }[node.depth ?? 1]
+
+      return [
+        ...parseLineBreakNodes(node, lastLineNumber, options),
+        {
+          type: getPluginType(options.editor, headingType),
+          children: remarkTransformElementChildren(
+            node,
+            lastLineNumber,
+            options
+          ),
+        },
+      ]
+    },
+  },
+  list: {
+    transform: (node, lastLineNumber, options) => {
+      if (options.indentList) {
+        const listStyleType = node.ordered ? 'decimal' : 'disc'
+
+        const parseListItems = (
+          _node: MdastNode,
+          listItems: TElement[] = [],
+          indent = 1
+        ) => {
+          _node.children!.forEach((listItem) => {
+            const [paragraph, ...subLists] = listItem.children!
+
+            listItems.push({
+              type: getPluginType(options.editor, ELEMENT_PARAGRAPH),
+              listStyleType,
+              indent,
+              children: remarkTransformElementChildren(
+                paragraph || '',
+                lastLineNumber,
+                options
+              ),
+            })
+
+            subLists.forEach((subList) => {
+              parseListItems(subList, listItems, indent + 1)
+            })
+          })
+
+          return listItems
+        }
+
+        return [
+          ...parseLineBreakNodes(node, lastLineNumber, options),
+          parseListItems(node),
+        ]
+      } else {
+        return [
+          ...parseLineBreakNodes(node, lastLineNumber, options),
+          {
+            type: getPluginType(
+              options.editor,
+              node.ordered ? ELEMENT_OL : ELEMENT_UL
+            ),
+            children: remarkTransformElementChildren(
+              node,
+              lastLineNumber,
+              options
+            ),
+          },
+        ]
+      }
+    },
+  },
+  listItem: {
+    transform: (node, lastLineNumber, options) => ({
+      type: getPluginType(options.editor, ELEMENT_LI),
+      children: remarkTransformElementChildren(
+        node,
+        lastLineNumber,
+        options
+      ).map(
+        (child) =>
+          ({
+            ...child,
+            type:
+              child.type === getPluginType(options.editor, ELEMENT_PARAGRAPH)
+                ? getPluginType(options.editor, ELEMENT_LIC)
+                : child.type,
+          } as TDescendant)
+      ),
+    }),
+  },
+  paragraph: {
+    transform: (node, lastLineNumber, options) => {
+      const children = remarkTransformElementChildren(
+        node,
+        lastLineNumber,
+        options
+      )
+
+      const paragraphType = getPluginType(options.editor, ELEMENT_PARAGRAPH)
+      const splitBlockTypes = new Set([
+        getPluginType(options.editor, ELEMENT_IMAGE),
+      ])
+
+      const elements: TElement[] = []
+      let inlineNodes: TDescendant[] = []
+
+      const flushInlineNodes = () => {
+        if (inlineNodes.length > 0) {
+          elements.push({
+            type: paragraphType,
+            children: inlineNodes,
+          })
+
+          inlineNodes = []
+        }
+      }
+
+      children.forEach((child) => {
+        const { type } = child
+
+        if (type && splitBlockTypes.has(type as string)) {
+          flushInlineNodes()
+          elements.push(child as TElement)
+        } else {
+          inlineNodes.push(child)
+        }
+      })
+
+      flushInlineNodes()
+
+      return [
+        ...parseLineBreakNodes(node, lastLineNumber, options),
+        ...elements,
+      ]
+    },
+  },
+  link: {
+    transform: (node, lastLineNumber, options) => [
+      ...parseLineBreakNodes(node, lastLineNumber, options),
+      {
+        type: getPluginType(options.editor, ELEMENT_LINK),
+        url: node.url,
+        children: remarkTransformElementChildren(node, lastLineNumber, options),
+      },
+    ],
+  },
+  image: {
+    transform: (node, lastLineNumber, options) => [
+      ...parseLineBreakNodes(node, lastLineNumber, options),
+      {
+        type: getPluginType(options.editor, ELEMENT_IMAGE),
+        children: [{ text: '' } as TText],
+        url: node.url,
+        caption: [{ text: node.alt } as TText],
+      },
+    ],
+  },
+  blockquote: {
+    transform: (node, lastLineNumber, options) => [
+      ...parseLineBreakNodes(node, lastLineNumber, options),
+      {
+        type: getPluginType(options.editor, ELEMENT_BLOCKQUOTE),
+        children: node.children!.flatMap((paragraph) =>
+          remarkTransformElementChildren(paragraph, lastLineNumber, options)
+        ),
+      },
+    ],
+  },
+  code: {
+    transform: (node, lastLineNumber, options) => [
+      ...parseLineBreakNodes(node, lastLineNumber, options),
+      {
+        type: getPluginType(options.editor, ELEMENT_CODE_BLOCK),
+        lang: node.lang ?? undefined,
+        children: (node.value || '').split('\n').map((line) => ({
+          type: getPluginType(options.editor, ELEMENT_CODE_LINE),
+          children: [{ text: line } as TText],
+        })),
+      },
+    ],
+  },
+  thematicBreak: {
+    transform: (node, lastLineNumber, options) => [
+      ...parseLineBreakNodes(node, lastLineNumber, options),
+      {
+        type: getPluginType(options.editor, ELEMENT_HR),
+        children: [{ text: '' } as TText],
+      },
+    ],
+  },
+}
+
+const parseLineBreakNodes = (
+  node: MdastNode,
+  lastLineNumber: number,
+  options: RemarkPluginOptions<Value>
+) => {
+  const lineBreaks = node.position.start.line - lastLineNumber
+
+  let lineBreakNodes = []
+
+  if (lineBreaks > 1)
+    lineBreakNodes.push(
+      ...Array(lineBreaks - 1).fill({
+        type: getPluginType(options.editor, ELEMENT_PARAGRAPH),
+        children: [{ text: '' } as TText],
+      })
+    )
+
+  return lineBreakNodes
+}

packages/lib/markdown/remark-slate/remarkDefaultTextRules.ts

@@ -0,0 +1,12 @@
+import { MARK_BOLD, MARK_CODE, MARK_ITALIC } from '@udecode/plate-basic-marks'
+import { getPluginType, Value } from '@udecode/plate-common'
+
+import { RemarkTextRules } from './types'
+
+export const remarkDefaultTextRules: RemarkTextRules<Value> = {
+  text: {},
+  emphasis: { mark: ({ editor }) => getPluginType(editor, MARK_ITALIC) },
+  strong: { mark: ({ editor }) => getPluginType(editor, MARK_BOLD) },
+  inlineCode: { mark: ({ editor }) => getPluginType(editor, MARK_CODE) },
+  html: { transform: (text: string) => text.replaceAll('<br>', '\n') },
+}

packages/lib/markdown/remark-slate/remarkPlugin.ts

@@ -0,0 +1,18 @@
+import { Value } from '@udecode/plate-common'
+
+import { remarkTransformNode } from './remarkTransformNode'
+import { MdastNode, RemarkPluginOptions } from './types'
+
+export function remarkPlugin<V extends Value>(options: RemarkPluginOptions<V>) {
+  let lastLineNumber = 1
+  const compiler = (node: { children: Array<MdastNode> }) => {
+    return node.children.flatMap((child) => {
+      const parsedChild = remarkTransformNode(child, lastLineNumber, options)
+      lastLineNumber = child.position?.end.line || lastLineNumber
+      return parsedChild
+    })
+  }
+
+  // @ts-ignore
+  this.Compiler = compiler
+}

packages/lib/markdown/remark-slate/remarkTextTypes.ts

@@ -0,0 +1,10 @@
+import { MdastNodeType } from './types'
+
+export const remarkTextTypes: MdastNodeType[] = [
+  'emphasis',
+  'strong',
+  'delete',
+  'inlineCode',
+  'html',
+  'text',
+]

packages/lib/markdown/remark-slate/remarkTransformElement.ts

@@ -0,0 +1,17 @@
+import { TElement, Value } from '@udecode/plate-common'
+
+import { MdastNode, RemarkPluginOptions } from './types'
+
+export const remarkTransformElement = <V extends Value>(
+  node: MdastNode,
+  lastLineNumber: number,
+  options: RemarkPluginOptions<V>
+): TElement | TElement[] => {
+  const { elementRules } = options
+
+  const { type } = node
+  const elementRule = (elementRules as any)[type!]
+  if (!elementRule) return []
+
+  return elementRule.transform(node, lastLineNumber, options)
+}

packages/lib/markdown/remark-slate/remarkTransformElementChildren.ts

@@ -0,0 +1,17 @@
+import { TDescendant, Value } from '@udecode/plate-common'
+
+import { remarkTransformNode } from './remarkTransformNode'
+import { MdastNode, RemarkPluginOptions } from './types'
+
+export const remarkTransformElementChildren = <V extends Value>(
+  node: MdastNode,
+  lastLineNumber: number,
+  options: RemarkPluginOptions<V>
+): TDescendant[] => {
+  const { children } = node
+  if (!children) return []
+
+  return children.flatMap((child) =>
+    remarkTransformNode(child, lastLineNumber, options)
+  )
+}

packages/lib/markdown/remark-slate/remarkTransformNode.ts

@@ -0,0 +1,20 @@
+import { TDescendant, Value } from '@udecode/plate-common'
+
+import { remarkTextTypes } from './remarkTextTypes'
+import { remarkTransformElement } from './remarkTransformElement'
+import { remarkTransformText } from './remarkTransformText'
+import { MdastNode, RemarkPluginOptions } from './types'
+
+export const remarkTransformNode = <V extends Value>(
+  node: MdastNode,
+  lastLineNumber: number,
+  options: RemarkPluginOptions<V>
+): TDescendant | TDescendant[] => {
+  const { type } = node
+
+  if (remarkTextTypes.includes(type!)) {
+    return remarkTransformText(node, options)
+  }
+
+  return remarkTransformElement(node, lastLineNumber, options)
+}

packages/lib/markdown/remark-slate/remarkTransformText.ts

@@ -0,0 +1,36 @@
+import { TText, Value } from '@udecode/plate-common'
+
+import { remarkDefaultTextRules } from './remarkDefaultTextRules'
+import { MdastNode, RemarkPluginOptions } from './types'
+
+export const remarkTransformText = <V extends Value>(
+  node: MdastNode,
+  options: RemarkPluginOptions<V>,
+  inheritedMarkProps: { [key: string]: boolean } = {}
+): TText | TText[] => {
+  const { editor, textRules } = options
+
+  const { type, value, children } = node
+  const textRule = (textRules as any)[type!] || remarkDefaultTextRules.text
+
+  const { mark, transform = (text: string) => text } = textRule
+
+  const markProps = mark
+    ? {
+        ...inheritedMarkProps,
+        [mark({ editor })]: true,
+      }
+    : inheritedMarkProps
+
+  const childTextNodes =
+    children?.flatMap((child) =>
+      remarkTransformText(child, options, markProps)
+    ) || []
+
+  const currentTextNodes =
+    value || childTextNodes.length === 0
+      ? [{ text: transform(value || ''), ...markProps } as TText]
+      : []
+
+  return [...currentTextNodes, ...childTextNodes]
+}

packages/lib/markdown/remark-slate/types.ts

@@ -0,0 +1,67 @@
+import { PlateEditor, TElement, Value } from '@udecode/plate-common'
+
+export type MdastElementType =
+  | 'paragraph'
+  | 'heading'
+  | 'list'
+  | 'listItem'
+  | 'link'
+  | 'image'
+  | 'blockquote'
+  | 'code'
+  | 'thematicBreak'
+
+export type MdastTextType =
+  | 'emphasis'
+  | 'strong'
+  | 'delete'
+  | 'inlineCode'
+  | 'html'
+  | 'text'
+
+export type MdastNodeType = MdastElementType | MdastTextType
+
+export interface MdastNode {
+  type?: MdastNodeType
+  ordered?: boolean
+  value?: string
+  text?: string
+  children?: Array<MdastNode>
+  depth?: 1 | 2 | 3 | 4 | 5 | 6
+  url?: string
+  alt?: string
+  lang?: string
+  // mdast metadata
+  position?: any
+  spread?: any
+  checked?: any
+  indent?: any
+}
+
+export type RemarkElementRule<V extends Value> = {
+  transform: (
+    node: MdastNode,
+    lastLineNumber: number,
+    options: RemarkPluginOptions<V>
+  ) => TElement | TElement[]
+}
+
+export type RemarkElementRules<V extends Value> = {
+  [key in MdastElementType]?: RemarkElementRule<V>
+}
+
+export type RemarkTextRule<V extends Value> = {
+  mark?: (options: RemarkPluginOptions<V>) => string
+  transform?: (text: string) => string
+}
+
+export type RemarkTextRules<V extends Value> = {
+  [key in MdastTextType]?: RemarkTextRule<V>
+}
+
+export type RemarkPluginOptions<V extends Value> = {
+  editor: PlateEditor<V>
+  elementRules: RemarkElementRules<V>
+  textRules: RemarkTextRules<V>
+  indentList?: boolean
+}

packages/lib/markdown/serializer/ast-types.ts

@@ -0,0 +1,243 @@
+export interface NodeTypes {
+  paragraph: string
+  blockquote: string
+  code_block: string
+  link: string
+  ul_list: string
+  ol_list: string
+  listItem: string
+  listItemChild: string
+  heading: {
+    1: string
+    2: string
+    3: string
+    4: string
+    5: string
+    6: string
+  }
+  emphasis_mark: string
+  strong_mark: string
+  delete_mark: string
+  inline_code_mark: string
+  thematic_break: string
+  image: string
+  variable: string
+  ['inline-variable']: string
+}
+
+export type MdastNodeType =
+  | 'paragraph'
+  | 'heading'
+  | 'list'
+  | 'listItem'
+  | 'link'
+  | 'image'
+  | 'blockquote'
+  | 'code'
+  | 'html'
+  | 'emphasis'
+  | 'strong'
+  | 'delete'
+  | 'inlineCode'
+  | 'thematicBreak'
+  | 'text'
+
+export const defaultNodeTypes: NodeTypes = {
+  paragraph: 'p',
+  blockquote: 'blockquote',
+  code_block: 'code_block',
+  link: 'a',
+  ul_list: 'ul',
+  ol_list: 'ol',
+  listItem: 'li',
+  listItemChild: 'lic',
+  heading: {
+    1: 'h1',
+    2: 'h2',
+    3: 'h3',
+    4: 'h4',
+    5: 'h5',
+    6: 'h6',
+  },
+  emphasis_mark: 'italic',
+  strong_mark: 'bold',
+  delete_mark: 'strikeThrough',
+  inline_code_mark: 'code',
+  thematic_break: 'thematic_break',
+  image: 'image',
+  variable: 'variable',
+  ['inline-variable']: 'inline-variable',
+}
+
+export interface LeafType {
+  text: string
+  strikeThrough?: boolean
+  bold?: boolean
+  italic?: boolean
+  code?: boolean
+  parentType?: string
+}
+
+export interface BlockType {
+  type: string
+  parentType?: string
+  link?: string
+  caption?: string
+  language?: string
+  break?: boolean
+  listIndex?: number
+  children: Array<BlockType | LeafType>
+}
+
+export interface InputNodeTypes {
+  paragraph: string
+  block_quote: string
+  code_block: string
+  link: string
+  ul_list: string
+  ol_list: string
+  listItem: string
+  heading: {
+    1: string
+    2: string
+    3: string
+    4: string
+    5: string
+    6: string
+  }
+  emphasis_mark: string
+  strong_mark: string
+  delete_mark: string
+  inline_code_mark: string
+  thematic_break: string
+  image: string
+}
+
+type RecursivePartial<T> = {
+  [P in keyof T]?: RecursivePartial<T[P]>
+}
+
+export interface OptionType<T extends InputNodeTypes = InputNodeTypes> {
+  nodeTypes?: RecursivePartial<T>
+  linkDestinationKey?: string
+  imageSourceKey?: string
+  imageCaptionKey?: string
+}
+
+export interface MdastNode {
+  type?: MdastNodeType
+  ordered?: boolean
+  value?: string
+  text?: string
+  children?: Array<MdastNode>
+  depth?: 1 | 2 | 3 | 4 | 5 | 6
+  url?: string
+  alt?: string
+  lang?: string
+  // mdast metadata
+  position?: any
+  spread?: any
+  checked?: any
+  indent?: any
+}
+
+export type TextNode = { text?: string | undefined }
+
+export type CodeBlockNode<T extends InputNodeTypes> = {
+  type: T['code_block']
+  language: string | undefined
+  children: Array<TextNode>
+}
+
+export type HeadingNode<T extends InputNodeTypes> = {
+  type:
+    | T['heading'][1]
+    | T['heading'][2]
+    | T['heading'][3]
+    | T['heading'][4]
+    | T['heading'][5]
+    | T['heading'][6]
+  children: Array<DeserializedNode<T>>
+}
+
+export type ListNode<T extends InputNodeTypes> = {
+  type: T['ol_list'] | T['ul_list']
+  children: Array<DeserializedNode<T>>
+}
+
+export type ListItemNode<T extends InputNodeTypes> = {
+  type: T['listItem']
+  children: Array<DeserializedNode<T>>
+}
+
+export type ParagraphNode<T extends InputNodeTypes> = {
+  type: T['paragraph']
+  break?: true
+  children: Array<DeserializedNode<T>>
+}
+
+export type LinkNode<T extends InputNodeTypes> = {
+  type: T['link']
+  children: Array<DeserializedNode<T>>
+  [urlKey: string]: string | undefined | Array<DeserializedNode<T>>
+}
+
+export type ImageNode<T extends InputNodeTypes> = {
+  type: T['image']
+  children: Array<DeserializedNode<T>>
+  [sourceOrCaptionKey: string]: string | undefined | Array<DeserializedNode<T>>
+}
+
+export type BlockQuoteNode<T extends InputNodeTypes> = {
+  type: T['block_quote']
+  children: Array<DeserializedNode<T>>
+}
+
+export type InlineCodeMarkNode<T extends InputNodeTypes> = {
+  type: T['inline_code_mark']
+  children: Array<TextNode>
+  language: string | undefined
+}
+
+export type ThematicBreakNode<T extends InputNodeTypes> = {
+  type: T['thematic_break']
+  children: Array<DeserializedNode<T>>
+}
+
+export type ItalicNode<T extends InputNodeTypes> = {
+  [K in T['emphasis_mark']]: true
+} & {
+  children: TextNode
+}
+
+export type BoldNode = {
+  bold: true
+  children: TextNode
+}
+
+export type StrikeThoughNode = {
+  strikeThrough: true
+  children: TextNode
+}
+
+export type InlineCodeNode = {
+  code: true
+  text: string | undefined
+}
+
+export type DeserializedNode<T extends InputNodeTypes> =
+  | CodeBlockNode<T>
+  | HeadingNode<T>
+  | ListNode<T>
+  | ListItemNode<T>
+  | ParagraphNode<T>
+  | LinkNode<T>
+  | ImageNode<T>
+  | BlockQuoteNode<T>
+  | InlineCodeMarkNode<T>
+  | ThematicBreakNode<T>
+  | ItalicNode<T>
+  | BoldNode
+  | StrikeThoughNode
+  | InlineCodeNode
+  | TextNode

packages/lib/markdown/serializer/serialize.ts

@@ -0,0 +1,286 @@
+import { BlockType, defaultNodeTypes, LeafType, NodeTypes } from './ast-types'
+import escapeHtml from 'escape-html'
+
+interface Options {
+  nodeTypes: NodeTypes
+  flavour?: 'common' | 'whatsapp'
+  listDepth?: number
+  ignoreParagraphNewline?: boolean
+}
+
+const isLeafNode = (node: BlockType | LeafType): node is LeafType => {
+  return typeof (node as LeafType).text === 'string'
+}
+
+const VOID_ELEMENTS: Array<keyof NodeTypes> = ['thematic_break', 'image']
+
+const BREAK_TAG = '<br>'
+
+export default function serialize(
+  chunk: BlockType | LeafType,
+  opts: Options = {
+    nodeTypes: defaultNodeTypes,
+  }
+): string | undefined {
+  const {
+    nodeTypes: userNodeTypes = defaultNodeTypes,
+    ignoreParagraphNewline = false,
+    listDepth = -1,
+  } = opts
+
+  let text = (chunk as LeafType).text || ''
+  let type = (chunk as BlockType).type || ''
+
+  const nodeTypes: NodeTypes = {
+    ...defaultNodeTypes,
+    ...userNodeTypes,
+    heading: {
+      ...defaultNodeTypes.heading,
+      ...userNodeTypes.heading,
+    },
+  }
+
+  if ('type' in chunk && chunk.type === nodeTypes['inline-variable'])
+    return chunk.children
+      .map((child) =>
+        serialize(
+          {
+            ...child,
+            parentType: nodeTypes['inline-variable'],
+          },
+          opts
+        )
+      )
+      .join('')
+
+  const LIST_TYPES = [nodeTypes.ul_list, nodeTypes.ol_list]
+
+  let children = text
+
+  if (!isLeafNode(chunk)) {
+    children = chunk.children
+      .map((c: BlockType | LeafType, index) => {
+        const isList = !isLeafNode(c)
+          ? (LIST_TYPES as string[]).includes(c.type || '')
+          : false
+
+        const selfIsList = (LIST_TYPES as string[]).includes(chunk.type || '')
+
+        // Links can have the following shape
+        // In which case we don't want to surround
+        // with break tags
+        // {
+        //  type: 'paragraph',
+        //  children: [
+        //    { text: '' },
+        //    { type: 'link', children: [{ text: foo.com }]}
+        //    { text: '' }
+        //  ]
+        // }
+        let childrenHasLink = false
+
+        if (!isLeafNode(chunk) && Array.isArray(chunk.children)) {
+          childrenHasLink = chunk.children.some(
+            (f) => !isLeafNode(f) && f.type === nodeTypes.link
+          )
+        }
+
+        return serialize(
+          {
+            ...c,
+            parentType: type === nodeTypes.listItem ? chunk.parentType : type,
+            listIndex:
+              type === nodeTypes.listItem ? chunk.listIndex : index + 1,
+          },
+          {
+            flavour: opts.flavour,
+            nodeTypes,
+            // WOAH.
+            // what we're doing here is pretty tricky, it relates to the block below where
+            // we check for ignoreParagraphNewline and set type to paragraph.
+            // We want to strip out empty paragraphs sometimes, but other times we don't.
+            // If we're the descendant of a list, we know we don't want a bunch
+            // of whitespace. If we're parallel to a link we also don't want
+            // to respect neighboring paragraphs
+            ignoreParagraphNewline:
+              (ignoreParagraphNewline ||
+                isList ||
+                selfIsList ||
+                childrenHasLink) &&
+              // if we have c.break, never ignore empty paragraph new line
+              !(c as BlockType).break,
+
+            // track depth of nested lists so we can add proper spacing
+            listDepth: (LIST_TYPES as string[]).includes(
+              (c as BlockType).type || ''
+            )
+              ? listDepth + 1
+              : listDepth,
+          }
+        )
+      })
+      .join('')
+  }
+
+  // This is pretty fragile code, check the long comment where we iterate over children
+  if (
+    !ignoreParagraphNewline &&
+    (text === '' || text === '\n') &&
+    chunk.parentType === nodeTypes.paragraph
+  ) {
+    type = nodeTypes.paragraph
+    children = BREAK_TAG
+  }
+
+  if (children === '' && !VOID_ELEMENTS.find((k) => nodeTypes[k] === type))
+    return
+
+  // Never allow decorating break tags with rich text formatting,
+  // this can malform generated markdown
+  // Also ensure we're only ever applying text formatting to leaf node
+  // level chunks, otherwise we can end up in a situation where
+  // we try applying formatting like to a node like this:
+  // "Text foo bar **baz**" resulting in "**Text foo bar **baz****"
+  // which is invalid markup and can mess everything up
+  if (children !== BREAK_TAG && isLeafNode(chunk)) {
+    if (chunk.strikeThrough && chunk.bold && chunk.italic) {
+      if (opts.flavour === 'whatsapp') {
+        children = retainWhitespaceAndFormat(children, '*_~')
+      } else {
+        children = retainWhitespaceAndFormat(children, '~~***')
+      }
+    } else if (chunk.bold && chunk.italic) {
+      if (opts.flavour === 'whatsapp') {
+        children = retainWhitespaceAndFormat(children, '*_')
+      } else {
+        children = retainWhitespaceAndFormat(children, '***')
+      }
+    } else {
+      if (chunk.bold) {
+        if (opts.flavour === 'whatsapp') {
+          children = retainWhitespaceAndFormat(children, '*')
+        } else {
+          children = retainWhitespaceAndFormat(children, '**')
+        }
+      }
+
+      if (chunk.italic) {
+        children = retainWhitespaceAndFormat(children, '_')
+      }
+
+      if (chunk.strikeThrough) {
+        children = retainWhitespaceAndFormat(children, '~~')
+      }
+
+      if (chunk.code) {
+        if (opts.flavour === 'whatsapp') {
+          children = retainWhitespaceAndFormat(children, '```')
+        } else {
+          children = retainWhitespaceAndFormat(children, '`')
+        }
+      }
+    }
+  }
+
+  if (chunk.parentType === nodeTypes['inline-variable']) {
+    if (opts.flavour === 'whatsapp') {
+      return children
+    }
+    return escapeHtml(children)
+  }
+
+  switch (type) {
+    case nodeTypes.heading[1]:
+      return `# ${children}\n`
+    case nodeTypes.heading[2]:
+      return `## ${children}\n`
+    case nodeTypes.heading[3]:
+      return `### ${children}\n`
+    case nodeTypes.heading[4]:
+      return `#### ${children}\n`
+    case nodeTypes.heading[5]:
+      return `##### ${children}\n`
+    case nodeTypes.heading[6]:
+      return `###### ${children}\n`
+
+    case nodeTypes.blockquote:
+      return `> ${children}\n`
+
+    case nodeTypes.code_block:
+      return `\`\`\`${
+        (chunk as BlockType).language || ''
+      }\n${children}\n\`\`\`\n`
+
+    case nodeTypes.link:
+      return `[${children}](${(chunk as any).url || ''})`
+    case nodeTypes.image:
+      return `![${(chunk as BlockType).caption}](${
+        (chunk as BlockType).link || ''
+      })`
+
+    case nodeTypes.listItemChild:
+      const isOL = chunk && chunk.parentType === nodeTypes.ol_list
+      const listIndex = 'listIndex' in chunk ? chunk.listIndex : undefined
+      const treatAsLeaf =
+        (chunk as BlockType).children.length === 1 &&
+        isLeafNode((chunk as BlockType).children[0])
+
+      let spacer = ''
+      for (let k = 0; listDepth > k; k++) {
+        if (isOL) {
+          // https://github.com/remarkjs/remark-react/issues/65
+          spacer += '   '
+        } else {
+          spacer += '  '
+        }
+      }
+      return `${spacer}${isOL ? `${listIndex}.` : '-'} ${children}${
+        treatAsLeaf ? '\n' : ''
+      }`
+
+    case nodeTypes.paragraph:
+      return `${children}\n`
+
+    case nodeTypes.thematic_break:
+      return `---\n`
+
+    default: {
+      if (opts.flavour === 'whatsapp') {
+        return children
+      }
+      return escapeHtml(children)
+    }
+  }
+}
+
+// This function handles the case of a string like this: "   foo   "
+// Where it would be invalid markdown to generate this: "**   foo   **"
+// We instead, want to trim the whitespace out, apply formatting, and then
+// bring the whitespace back. So our returned string looks like this: "   **foo**   "
+function retainWhitespaceAndFormat(string: string, format: string) {
+  // we keep this for a comparison later
+  const frozenString = string.trim()
+
+  // children will be mutated
+  let children = frozenString
+
+  // We reverse the right side formatting, to properly handle bold/italic and strikeThrough
+  // formats, so we can create ~~***FooBar***~~
+  const fullFormat = `${format}${children}${reverseStr(format)}`
+
+  // This conditions accounts for no whitespace in our string
+  // if we don't have any, we can return early.
+  if (children.length === string.length) {
+    return fullFormat
+  }
+
+  // if we do have whitespace, let's add our formatting around our trimmed string
+  // We reverse the right side formatting, to properly handle bold/italic and strikeThrough
+  // formats, so we can create ~~***FooBar***~~
+  const formattedString = format + children + reverseStr(format)
+
+  // and replace the non-whitespace content of the string
+  return string.replace(frozenString, formattedString)
+}
+
+const reverseStr = (string: string) => string.split('').reverse().join('')