blsmedia/bot
2
0
Fork 0
Code Activity

🐛 (ai) Enable multi urls for vision

Browse Source 
Closes #1764
This commit is contained in:
Baptiste Arnaud
2024-09-04 14:07:32 +02:00
parent fa14029fed
commit 3b87801c92
3 changed files with 66 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
packages/ai/splitUserTextMessageIntoBlocks.ts
View File

@@ -9,19 +9,13 @@ export const splitUserTextMessageIntoBlocks = async ({
input,
shouldDownloadImages,
}: Props): Promise<UserContent> => {
const urlRegex = /(^|\n\n)(https?:\/\/.+)(\n\n|$)/g
const match = input.match(urlRegex)
if (!match) return input
const splittedInput = input.split('\n\n')
let parts: (TextPart | ImagePart)[] = []
let processedInput = input
for (const url of match) {
const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url))
if (textBeforeUrl.trim().length > 0) {
parts.push({ type: 'text', text: textBeforeUrl })
}
for (const part of splittedInput) {
if (part.startsWith('http') || part.startsWith('["http')) {
const urls = part.startsWith('[') ? JSON.parse(part) : [part]
for (const url of urls) {
const cleanUrl = url.trim()
try {
const response = await ky.get(cleanUrl)
if (
@@ -44,13 +38,16 @@ export const splitUserTextMessageIntoBlocks = async ({
console.error(err)
}
}
processedInput = processedInput.slice(
processedInput.indexOf(url) + url.length
)
}
if (processedInput.trim().length > 0) {
parts.push({ type: 'text', text: processedInput })
} else {
if (parts.at(-1)?.type === 'text') {
const lastText = parts.at(-1) as TextPart
parts = parts.slice(0, -1)
parts.push({ type: 'text', text: lastText.text + '\n\n' + part })
} else {
parts.push({ type: 'text', text: part })
}
}
}
return parts

7
packages/bot-engine/blocks/logic/setVariable/executeSetVariable.ts
View File

@@ -298,13 +298,6 @@ const getExpressionToEvaluate =
type: 'value',
value: [item],
}
if (isEmpty(item))
return {
type: 'value',
value: Array.isArray(variableValue)
? variableValue
: [variableValue],
}
if (!Array.isArray(variableValue))
return { type: 'value', value: [variableValue, item] }
return { type: 'value', value: variableValue.concat(item) }

36
packages/forge/blocks/openai/helpers/splitUserTextMessageIntoOpenAIBlocks.ts
View File

@@ -4,19 +4,13 @@ import OpenAI from 'openai'
export const splitUserTextMessageIntoOpenAIBlocks = async (
input: string
): Promise<string | OpenAI.Chat.ChatCompletionContentPart[]> => {
const urlRegex = /(^|\n\n)(https?:\/\/[^\s]+)(\n\n|$)/g
const match = input.match(urlRegex)
if (!match) return input
const splittedInput = input.split('\n\n')
let parts: OpenAI.Chat.ChatCompletionContentPart[] = []
let processedInput = input
for (const url of match) {
const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url))
if (textBeforeUrl.trim().length > 0) {
parts.push({ type: 'text', text: textBeforeUrl })
}
for (const part of splittedInput) {
if (part.startsWith('http') || part.startsWith('["http')) {
const urls = part.startsWith('[') ? JSON.parse(part) : [part]
for (const url of urls) {
const cleanUrl = url.trim()
try {
const response = await ky.get(cleanUrl)
if (
@@ -27,10 +21,7 @@ export const splitUserTextMessageIntoOpenAIBlocks = async (
} else {
parts.push({
type: 'image_url',
image_url: {
url: url.trim(),
detail: 'auto',
},
image_url: url.trim(),
})
}
} catch (err) {
@@ -40,13 +31,16 @@ export const splitUserTextMessageIntoOpenAIBlocks = async (
console.error(err)
}
}
processedInput = processedInput.slice(
processedInput.indexOf(url) + url.length
)
}
if (processedInput.trim().length > 0) {
parts.push({ type: 'text', text: processedInput })
} else {
if (parts.at(-1)?.type === 'text') {
const lastText = parts.at(-1) as OpenAI.ChatCompletionContentPartText
parts = parts.slice(0, -1)
parts.push({ type: 'text', text: lastText.text + '\n\n' + part })
} else {
parts.push({ type: 'text', text: part })
}
}
}
return parts