blsmedia/bot
2
0
Fork 0
Code Activity

🐛 (ai) Enable multi urls for vision

Browse Source 
Closes #1764
This commit is contained in:
Baptiste Arnaud
2024-09-04 14:07:32 +02:00
parent fa14029fed
commit 3b87801c92
3 changed files with 66 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
packages/ai/splitUserTextMessageIntoBlocks.ts
View File

@ -9,19 +9,13 @@ export const splitUserTextMessageIntoBlocks = async ({
input, input,
shouldDownloadImages, shouldDownloadImages,
}: Props): Promise<UserContent> => { }: Props): Promise<UserContent> => {
const urlRegex = /(^|\n\n)(https?:\/\/.+)(\n\n|$)/g const splittedInput = input.split('\n\n')
const match = input.match(urlRegex)
if (!match) return input
let parts: (TextPart | ImagePart)[] = [] let parts: (TextPart | ImagePart)[] = []
let processedInput = input for (const part of splittedInput) {
if (part.startsWith('http') || part.startsWith('["http')) {
for (const url of match) { const urls = part.startsWith('[') ? JSON.parse(part) : [part]
const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url)) for (const url of urls) {
if (textBeforeUrl.trim().length > 0) {
parts.push({ type: 'text', text: textBeforeUrl })
}
const cleanUrl = url.trim() const cleanUrl = url.trim()
try { try {
const response = await ky.get(cleanUrl) const response = await ky.get(cleanUrl)
if ( if (
@ -44,13 +38,16 @@ export const splitUserTextMessageIntoBlocks = async ({
console.error(err) console.error(err)
} }
} }
processedInput = processedInput.slice(
processedInput.indexOf(url) + url.length
)
} }
} else {
if (processedInput.trim().length > 0) { if (parts.at(-1)?.type === 'text') {
parts.push({ type: 'text', text: processedInput }) const lastText = parts.at(-1) as TextPart
parts = parts.slice(0, -1)
parts.push({ type: 'text', text: lastText.text + '\n\n' + part })
} else {
parts.push({ type: 'text', text: part })
}
}
} }
return parts return parts

7
packages/bot-engine/blocks/logic/setVariable/executeSetVariable.ts
View File

@ -298,13 +298,6 @@ const getExpressionToEvaluate =
type: 'value', type: 'value',
value: [item], value: [item],
} }
if (isEmpty(item))
return {
type: 'value',
value: Array.isArray(variableValue)
? variableValue
: [variableValue],
}
if (!Array.isArray(variableValue)) if (!Array.isArray(variableValue))
return { type: 'value', value: [variableValue, item] } return { type: 'value', value: [variableValue, item] }
return { type: 'value', value: variableValue.concat(item) } return { type: 'value', value: variableValue.concat(item) }

36
packages/forge/blocks/openai/helpers/splitUserTextMessageIntoOpenAIBlocks.ts
View File

@ -4,19 +4,13 @@ import OpenAI from 'openai'
export const splitUserTextMessageIntoOpenAIBlocks = async ( export const splitUserTextMessageIntoOpenAIBlocks = async (
input: string input: string
): Promise<string | OpenAI.Chat.ChatCompletionContentPart[]> => { ): Promise<string | OpenAI.Chat.ChatCompletionContentPart[]> => {
const urlRegex = /(^|\n\n)(https?:\/\/[^\s]+)(\n\n|$)/g const splittedInput = input.split('\n\n')
const match = input.match(urlRegex)
if (!match) return input
let parts: OpenAI.Chat.ChatCompletionContentPart[] = [] let parts: OpenAI.Chat.ChatCompletionContentPart[] = []
let processedInput = input for (const part of splittedInput) {
if (part.startsWith('http') || part.startsWith('["http')) {
for (const url of match) { const urls = part.startsWith('[') ? JSON.parse(part) : [part]
const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url)) for (const url of urls) {
if (textBeforeUrl.trim().length > 0) {
parts.push({ type: 'text', text: textBeforeUrl })
}
const cleanUrl = url.trim() const cleanUrl = url.trim()
try { try {
const response = await ky.get(cleanUrl) const response = await ky.get(cleanUrl)
if ( if (
@ -27,10 +21,7 @@ export const splitUserTextMessageIntoOpenAIBlocks = async (
} else { } else {
parts.push({ parts.push({
type: 'image_url', type: 'image_url',
image_url: { image_url: url.trim(),
url: url.trim(),
detail: 'auto',
},
}) })
} }
} catch (err) { } catch (err) {
@ -40,13 +31,16 @@ export const splitUserTextMessageIntoOpenAIBlocks = async (
console.error(err) console.error(err)
} }
} }
processedInput = processedInput.slice(
processedInput.indexOf(url) + url.length
)
} }
} else {
if (processedInput.trim().length > 0) { if (parts.at(-1)?.type === 'text') {
parts.push({ type: 'text', text: processedInput }) const lastText = parts.at(-1) as OpenAI.ChatCompletionContentPartText
parts = parts.slice(0, -1)
parts.push({ type: 'text', text: lastText.text + '\n\n' + part })
} else {
parts.push({ type: 'text', text: part })
}
}
} }
return parts return parts