2
0

🧐 Add exportResults script

This commit is contained in:
Baptiste Arnaud
2024-01-12 10:16:01 +01:00
parent 5d088b1e64
commit 69b113fc85
32 changed files with 319 additions and 332 deletions

View File

@ -11,7 +11,6 @@ import {
import { createTransport } from 'nodemailer'
import Mail from 'nodemailer/lib/mailer'
import { byId, isDefined, isEmpty, isNotDefined, omit } from '@typebot.io/lib'
import { getDefinedVariables, parseAnswers } from '@typebot.io/lib/results'
import { decrypt } from '@typebot.io/lib/api/encryption/decrypt'
import { defaultFrom, defaultTransportOptions } from './constants'
import { findUniqueVariableValue } from '@typebot.io/variables/findUniqueVariableValue'
@ -20,6 +19,7 @@ import { ExecuteIntegrationResponse } from '../../../types'
import prisma from '@typebot.io/lib/prisma'
import { parseVariables } from '@typebot.io/variables/parseVariables'
import { defaultSendEmailOptions } from '@typebot.io/schemas/features/blocks/integrations/sendEmail/constants'
import { parseAnswers } from '@typebot.io/lib/results/parseAnswers'
export const sendEmailSuccessDescription = 'Email successfully sent'
export const sendEmailErrorDescription = 'Email not sent'
@ -248,7 +248,7 @@ const getEmailBody = async ({
text: !isBodyCode ? body : undefined,
}
const answers = parseAnswers({
variables: getDefinedVariables(typebot.variables),
variables: typebot.variables,
answers: answersInSession,
})
return {

View File

@ -14,7 +14,6 @@ import {
} from '@typebot.io/schemas'
import { stringify } from 'qs'
import { isDefined, isEmpty, isNotDefined, omit } from '@typebot.io/lib'
import { getDefinedVariables, parseAnswers } from '@typebot.io/lib/results'
import got, { Method, HTTPError, OptionsInit } from 'got'
import { resumeWebhookExecution } from './resumeWebhookExecution'
import { ExecuteIntegrationResponse } from '../../../types'
@ -27,6 +26,7 @@ import {
maxTimeout,
} from '@typebot.io/schemas/features/blocks/integrations/webhook/constants'
import { env } from '@typebot.io/env'
import { parseAnswers } from '@typebot.io/lib/results/parseAnswers'
type ParsedWebhook = ExecutableWebhook & {
basicAuth: { username?: string; password?: string }
@ -297,7 +297,7 @@ const getBodyContent = async ({
? JSON.stringify(
parseAnswers({
answers,
variables: getDefinedVariables(variables),
variables,
})
)
: body ?? undefined

View File

@ -8,9 +8,9 @@ import {
Variable,
} from '@typebot.io/schemas'
import { isInputBlock, byId, isNotDefined } from '@typebot.io/lib'
import { parseResultHeader } from '@typebot.io/lib/results'
import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants'
import { LogicBlockType } from '@typebot.io/schemas/features/blocks/logic/constants'
import { parseResultHeader } from '@typebot.io/lib/results/parseResultHeader'
export const parseSampleResult =
(

View File

@ -7,10 +7,10 @@ import {
import got from 'got'
import { decrypt } from '@typebot.io/lib/api/encryption/decrypt'
import { byId, isDefined, isEmpty } from '@typebot.io/lib'
import { getDefinedVariables, parseAnswers } from '@typebot.io/lib/results'
import prisma from '@typebot.io/lib/prisma'
import { ExecuteIntegrationResponse } from '../../../types'
import { updateVariablesInSession } from '@typebot.io/variables/updateVariablesInSession'
import { parseAnswers } from '@typebot.io/lib/results/parseAnswers'
const URL = 'https://api.zemantic.ai/v1/search-documents'
@ -50,7 +50,7 @@ export const executeZemanticAiBlock = async (
const { typebot, answers } = newSessionState.typebotsQueue[0]
const templateVars = parseAnswers({
variables: getDefinedVariables(typebot.variables),
variables: typebot.variables,
answers: answers,
})

View File

@ -1,5 +1,4 @@
import prisma from '@typebot.io/lib/prisma'
import { getDefinedVariables } from '@typebot.io/lib/results'
import { TypebotInSession } from '@typebot.io/schemas'
type Props = {
@ -27,7 +26,7 @@ export const createResultIfNotExist = async ({
typebotId: typebot.id,
isCompleted: isCompleted ? true : false,
hasStarted,
variables: getDefinedVariables(typebot.variables),
variables: typebot.variables,
},
],
})

View File

@ -1,6 +1,6 @@
import prisma from '@typebot.io/lib/prisma'
import { getDefinedVariables } from '@typebot.io/lib/results'
import { TypebotInSession } from '@typebot.io/schemas'
import { filterVariablesWithValues } from '@typebot.io/variables/filterVariablesWithValues'
type Props = {
resultId: string
@ -18,7 +18,7 @@ export const upsertResult = async ({
where: { id: resultId },
select: { id: true },
})
const variablesWithValue = getDefinedVariables(typebot.variables)
const variablesWithValue = filterVariablesWithValues(typebot.variables)
if (existingResult) {
return prisma.result.updateMany({

View File

@ -0,0 +1,8 @@
export const parseUniqueKey = (
key: string,
existingKeys: string[],
count = 0
): string => {
if (!existingKeys.includes(key)) return key
return parseUniqueKey(`${key} (${count + 1})`, existingKeys, count + 1)
}

View File

@ -0,0 +1,79 @@
import {
ResultWithAnswers,
ResultHeaderCell,
VariableWithValue,
Answer,
TableData,
} from '@typebot.io/schemas'
import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants'
import { isDefined } from '../utils'
type CellParser = (
content: VariableWithValue['value'],
blockType?: InputBlockType
) => { element?: React.JSX.Element; plainText: string }
const defaultCellParser: CellParser = (content, blockType) => {
if (!content) return { plainText: '' }
if (Array.isArray(content))
return {
plainText: content.join(', '),
}
return blockType === InputBlockType.FILE
? { plainText: content }
: { plainText: content.toString() }
}
export const convertResultsToTableData = (
results: ResultWithAnswers[] | undefined,
headerCells: ResultHeaderCell[],
cellParser: CellParser = defaultCellParser
): TableData[] =>
(results ?? []).map((result) => ({
id: { plainText: result.id },
date: {
plainText: convertDateToReadable(result.createdAt),
},
...[...result.answers, ...result.variables].reduce<{
[key: string]: { element?: JSX.Element; plainText: string }
}>((tableData, answerOrVariable) => {
if ('groupId' in answerOrVariable) {
const answer = answerOrVariable satisfies Answer
const header = answer.variableId
? headerCells.find((headerCell) =>
headerCell.variableIds?.includes(answer.variableId as string)
)
: headerCells.find((headerCell) =>
headerCell.blocks?.some((block) => block.id === answer.blockId)
)
if (!header || !header.blocks || !header.blockType) return tableData
const variableValue = result.variables.find(
(variable) => variable.id === answer.variableId
)?.value
const content = variableValue ?? answer.content
return {
...tableData,
[header.id]: cellParser(content, header.blockType),
}
}
const variable = answerOrVariable satisfies VariableWithValue
if (variable.value === null) return tableData
const headerId = headerCells.find((headerCell) =>
headerCell.variableIds?.includes(variable.id)
)?.id
if (!headerId) return tableData
if (isDefined(tableData[headerId])) return tableData
return {
...tableData,
[headerId]: cellParser(variable.value),
}
}, {}),
}))
const convertDateToReadable = (date: Date): string =>
date.toDateString().split(' ').slice(1, 3).join(' ') +
', ' +
date.toLocaleTimeString([], {
hour: '2-digit',
minute: '2-digit',
})

View File

@ -0,0 +1,39 @@
import {
AnswerInSessionState,
Variable,
VariableWithValue,
} from '@typebot.io/schemas'
import { isDefined, isEmpty } from '../utils'
export const parseAnswers = ({
answers,
variables: resultVariables,
}: {
answers: AnswerInSessionState[]
variables: Variable[]
}): {
[key: string]: string
} => {
const variablesWithValues = resultVariables.filter((variable) =>
isDefined(variable.value)
) as VariableWithValue[]
return {
submittedAt: new Date().toISOString(),
...[...answers, ...variablesWithValues].reduce<{
[key: string]: string
}>((o, answerOrVariable) => {
if ('id' in answerOrVariable) {
const variable = answerOrVariable
if (variable.value === null) return o
return { ...o, [variable.name]: variable.value.toString() }
}
const answer = answerOrVariable as AnswerInSessionState
if (isEmpty(answer.key)) return o
return {
...o,
[answer.key]: answer.value,
}
}, {}),
}
}

View File

@ -0,0 +1,15 @@
import { ResultHeaderCell } from '@typebot.io/schemas'
export const parseColumnsOrder = (
existingOrder: string[] | undefined,
resultHeader: ResultHeaderCell[]
) =>
existingOrder
? [
...existingOrder.slice(0, -1),
...resultHeader
.filter((header) => !existingOrder.includes(header.id))
.map((h) => h.id),
'logs',
]
: ['select', ...resultHeader.map((h) => h.id), 'logs']

View File

@ -1,15 +1,13 @@
import {
ResultWithAnswers,
ResultHeaderCell,
Group,
Variable,
InputBlock,
ResultHeaderCell,
VariableWithValue,
Typebot,
ResultWithAnswers,
AnswerInSessionState,
} from '@typebot.io/schemas'
import { isInputBlock, isDefined, byId, isNotEmpty, isEmpty } from './utils'
import { InputBlockType } from '@typebot.io/schemas/features/blocks/inputs/constants'
import { isInputBlock, byId, isNotEmpty } from '../utils'
export const parseResultHeader = (
typebot: Pick<Typebot, 'groups' | 'variables'>,
@ -212,37 +210,3 @@ const parseResultsFromPreviousBotVersions = (
},
]
}, [])
export const parseAnswers = ({
answers,
variables: resultVariables,
}: {
answers: AnswerInSessionState[]
variables: VariableWithValue[]
}): {
[key: string]: string
} => {
return {
submittedAt: new Date().toISOString(),
...[...answers, ...resultVariables].reduce<{
[key: string]: string
}>((o, answerOrVariable) => {
if ('id' in answerOrVariable) {
const variable = answerOrVariable
if (variable.value === null) return o
return { ...o, [variable.name]: variable.value.toString() }
}
const answer = answerOrVariable as AnswerInSessionState
if (isEmpty(answer.key)) return o
return {
...o,
[answer.key]: answer.value,
}
}, {}),
}
}
export const getDefinedVariables = (variables: Variable[]) =>
variables.filter((variable) =>
isDefined(variable.value)
) as VariableWithValue[]

View File

@ -72,3 +72,9 @@ export type ResultHeaderCell = {
blockType?: InputBlockType
variableIds?: string[]
}
export type CellValueType = { element?: JSX.Element; plainText: string }
export type TableData = {
id: Pick<CellValueType, 'plainText'>
} & Record<string, CellValueType>

View File

@ -0,0 +1,119 @@
import { PrismaClient } from '@typebot.io/prisma'
import * as p from '@clack/prompts'
import { promptAndSetEnvironment } from './utils'
import cliProgress from 'cli-progress'
import { writeFileSync } from 'fs'
import {
ResultWithAnswers,
Typebot,
resultWithAnswersSchema,
} from '@typebot.io/schemas'
import { byId } from '@typebot.io/lib'
import { parseResultHeader } from '@typebot.io/lib/results/parseResultHeader'
import { convertResultsToTableData } from '@typebot.io/lib/results/convertResultsToTableData'
import { parseColumnsOrder } from '@typebot.io/lib/results/parseColumnsOrder'
import { parseUniqueKey } from '@typebot.io/lib/parseUniqueKey'
import { unparse } from 'papaparse'
import { z } from 'zod'
const exportResults = async () => {
await promptAndSetEnvironment('production')
const prisma = new PrismaClient()
const typebotId = (await p.text({
message: 'Typebot ID?',
})) as string
if (!typebotId || typeof typebotId !== 'string') {
console.log('No id provided')
return
}
const progressBar = new cliProgress.SingleBar(
{},
cliProgress.Presets.shades_classic
)
const typebot = (await prisma.typebot.findUnique({
where: {
id: typebotId,
},
})) as Typebot | null
if (!typebot) {
console.log('No typebot found')
return
}
const totalResultsToExport = await prisma.result.count({
where: {
typebotId,
hasStarted: true,
isArchived: false,
},
})
progressBar.start(totalResultsToExport, 0)
const results: ResultWithAnswers[] = []
for (let skip = 0; skip < totalResultsToExport; skip += 50) {
results.push(
...z.array(resultWithAnswersSchema).parse(
await prisma.result.findMany({
take: 50,
skip,
where: {
typebotId,
hasStarted: true,
isArchived: false,
},
orderBy: {
createdAt: 'desc',
},
include: { answers: true },
})
)
)
progressBar.increment(50)
}
progressBar.stop()
writeFileSync('logs/results.json', JSON.stringify(results))
const resultHeader = parseResultHeader(typebot, [])
const dataToUnparse = convertResultsToTableData(results, resultHeader)
const headerIds = parseColumnsOrder(
typebot?.resultsTablePreferences?.columnsOrder,
resultHeader
).reduce<string[]>((currentHeaderIds, columnId) => {
if (typebot?.resultsTablePreferences?.columnsVisibility[columnId] === false)
return currentHeaderIds
const columnLabel = resultHeader.find(
(headerCell) => headerCell.id === columnId
)?.id
if (!columnLabel) return currentHeaderIds
return [...currentHeaderIds, columnLabel]
}, [])
const data = dataToUnparse.map<{ [key: string]: string }>((data) => {
const newObject: { [key: string]: string } = {}
headerIds?.forEach((headerId) => {
const headerLabel = resultHeader.find(byId(headerId))?.label
if (!headerLabel) return
const newKey = parseUniqueKey(headerLabel, Object.keys(newObject))
newObject[newKey] = data[headerId]?.plainText
})
return newObject
})
const csv = unparse(data)
writeFileSync('logs/results.csv', csv)
}
exportResults()

View File

@ -1,203 +0,0 @@
import { PrismaClient } from '@typebot.io/prisma'
import { writeFileSync } from 'fs'
import {
Block,
BlockOptions,
BlockType,
defaultEmailInputOptions,
Group,
InputBlockType,
PublicTypebot,
publicTypebotSchema,
Theme,
Typebot,
} from '@typebot.io/schemas'
import { isDefined, isNotDefined } from '@typebot.io/lib'
import { promptAndSetEnvironment } from './utils'
import { detailedDiff } from 'deep-object-diff'
const fixTypebot = (brokenTypebot: Typebot | PublicTypebot) =>
({
...brokenTypebot,
theme: fixTheme(brokenTypebot.theme),
groups: fixGroups(brokenTypebot.groups),
} satisfies Typebot | PublicTypebot)
const fixTheme = (brokenTheme: Theme) =>
({
...brokenTheme,
chat: {
...brokenTheme.chat,
hostAvatar: brokenTheme.chat.hostAvatar
? {
isEnabled: brokenTheme.chat.hostAvatar.isEnabled,
url: brokenTheme.chat.hostAvatar.url ?? undefined,
}
: undefined,
},
} satisfies Theme)
const fixGroups = (brokenGroups: Group[]) =>
brokenGroups.map(
(brokenGroup, index) =>
({
...brokenGroup,
graphCoordinates: {
...brokenGroup.graphCoordinates,
x: brokenGroup.graphCoordinates.x ?? 0,
y: brokenGroup.graphCoordinates.y ?? 0,
},
blocks: fixBlocks(brokenGroup.blocks, brokenGroup.id, index),
} satisfies Group)
)
const fixBlocks = (
brokenBlocks: Block[],
groupId: string,
groupIndex: number
) => {
if (groupIndex === 0 && brokenBlocks.length > 1) return [brokenBlocks[0]]
return brokenBlocks
.filter((block) => block && Object.keys(block).length > 0)
.map((brokenBlock) => {
return removeUndefinedFromObject({
...brokenBlock,
webhookId:
('webhookId' in brokenBlock ? brokenBlock.webhookId : undefined) ??
('webhook' in brokenBlock && brokenBlock.webhook
? //@ts-ignore
brokenBlock.webhook.id
: undefined),
webhook: undefined,
groupId: brokenBlock.groupId ?? groupId,
options:
brokenBlock && 'options' in brokenBlock && brokenBlock.options
? fixBrokenBlockOption(brokenBlock.options, brokenBlock.type)
: undefined,
})
}) as Block[]
}
const fixBrokenBlockOption = (options: BlockOptions, blockType: BlockType) =>
removeUndefinedFromObject({
...options,
sheetId:
'sheetId' in options && isDefined(options.sheetId)
? options.sheetId.toString()
: undefined,
step:
'step' in options && isDefined(options.step) ? options.step : undefined,
value:
'value' in options && isDefined(options.value)
? options.value
: undefined,
retryMessageContent: fixRetryMessageContent(
//@ts-ignore
options.retryMessageContent,
blockType
),
}) as BlockOptions
const fixRetryMessageContent = (
retryMessageContent: string | undefined,
blockType: BlockType
) => {
if (isNotDefined(retryMessageContent) && blockType === InputBlockType.EMAIL)
return defaultEmailInputOptions.retryMessageContent
if (isNotDefined(retryMessageContent)) return undefined
return retryMessageContent
}
const removeUndefinedFromObject = (obj: any) => {
Object.keys(obj).forEach((key) => obj[key] === undefined && delete obj[key])
return obj
}
const resolve = (path: string, obj: object, separator = '.') => {
const properties = Array.isArray(path) ? path : path.split(separator)
//@ts-ignore
return properties.reduce((prev, curr) => prev?.[curr], obj)
}
const fixTypebots = async () => {
await promptAndSetEnvironment()
const prisma = new PrismaClient({
log: [{ emit: 'event', level: 'query' }, 'info', 'warn', 'error'],
})
const typebots = await prisma.publicTypebot.findMany({
where: {
updatedAt: {
gte: new Date('2023-01-01T00:00:00.000Z'),
},
},
})
writeFileSync('logs/typebots.json', JSON.stringify(typebots))
const total = typebots.length
let totalFixed = 0
let progress = 0
const fixedTypebots: (Typebot | PublicTypebot)[] = []
const diffs: any[] = []
for (const typebot of typebots) {
progress += 1
console.log(
`Progress: ${progress}/${total} (${Math.round(
(progress / total) * 100
)}%) (${totalFixed} fixed typebots)`
)
const parser = publicTypebotSchema.safeParse({
...typebot,
updatedAt: new Date(typebot.updatedAt),
createdAt: new Date(typebot.createdAt),
})
if ('error' in parser) {
const fixedTypebot = {
...fixTypebot(typebot as Typebot | PublicTypebot),
updatedAt: new Date(),
createdAt: new Date(typebot.createdAt),
}
publicTypebotSchema.parse(fixedTypebot)
fixedTypebots.push(fixedTypebot)
totalFixed += 1
diffs.push({
id: typebot.id,
failedObject: resolve(parser.error.issues[0].path.join('.'), typebot),
...detailedDiff(typebot, fixedTypebot),
})
}
}
writeFileSync('logs/fixedTypebots.json', JSON.stringify(fixedTypebots))
writeFileSync(
'logs/diffs.json',
JSON.stringify(diffs.reverse().slice(0, 100))
)
const queries = fixedTypebots.map((fixedTypebot) =>
prisma.publicTypebot.updateMany({
where: { id: fixedTypebot.id },
data: {
...fixedTypebot,
// theme: fixedTypebot.theme ?? undefined,
// settings: fixedTypebot.settings ?? undefined,
// resultsTablePreferences:
// 'resultsTablePreferences' in fixedTypebot &&
// fixedTypebot.resultsTablePreferences
// ? fixedTypebot.resultsTablePreferences
// : undefined,
} as any,
})
)
const totalQueries = queries.length
progress = 0
prisma.$on('query', () => {
progress += 1
console.log(`Progress: ${progress}/${totalQueries}`)
})
await prisma.$transaction(queries)
}
fixTypebots()

View File

@ -25,14 +25,17 @@
"updateWorkspace": "tsx updateWorkspace.ts",
"inspectTypebot": "tsx inspectTypebot.ts",
"inspectWorkspace": "tsx inspectWorkspace.ts",
"getCoupon": "tsx getCoupon.ts"
"getCoupon": "tsx getCoupon.ts",
"exportResults": "tsx exportResults.ts"
},
"devDependencies": {
"@typebot.io/emails": "workspace:*",
"@typebot.io/lib": "workspace:*",
"@typebot.io/prisma": "workspace:*",
"@typebot.io/schemas": "workspace:*",
"@types/cli-progress": "^3.11.5",
"@types/node": "20.4.2",
"@types/papaparse": "5.3.7",
"@types/prompts": "2.4.4",
"deep-object-diff": "1.1.9",
"got": "12.6.0",
@ -44,6 +47,8 @@
},
"dependencies": {
"@clack/prompts": "^0.7.0",
"@paralleldrive/cuid2": "2.2.1"
"@paralleldrive/cuid2": "2.2.1",
"cli-progress": "^3.12.0",
"papaparse": "5.4.1"
}
}

View File

@ -0,0 +1,9 @@
import { isDefined } from '@typebot.io/lib'
import { Variable, VariableWithValue } from '../schemas'
export const filterVariablesWithValues = (
variables: Variable[]
): VariableWithValue[] =>
variables.filter((variable) =>
isDefined(variable.value)
) as VariableWithValue[]