blsmedia/bot
2
0
Fork 0
Code Activity

🐛 (ai) Enable multi urls for vision

Browse Source 
Closes #1764
This commit is contained in:
Baptiste Arnaud
2024-09-04 14:07:32 +02:00
parent fa14029fed
commit 3b87801c92
3 changed files with 66 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
packages/ai/splitUserTextMessageIntoBlocks.ts
View File

@ -9,48 +9,45 @@ export const splitUserTextMessageIntoBlocks = async ({
input, input,
shouldDownloadImages, shouldDownloadImages,
}: Props): Promise<UserContent> => { }: Props): Promise<UserContent> => {
const urlRegex = /(^|\n\n)(https?:\/\/.+)(\n\n|$)/g const splittedInput = input.split('\n\n')
const match = input.match(urlRegex)
if (!match) return input
let parts: (TextPart | ImagePart)[] = [] let parts: (TextPart | ImagePart)[] = []
let processedInput = input for (const part of splittedInput) {
if (part.startsWith('http') || part.startsWith('["http')) {
for (const url of match) { const urls = part.startsWith('[') ? JSON.parse(part) : [part]
const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url)) for (const url of urls) {
if (textBeforeUrl.trim().length > 0) { const cleanUrl = url.trim()
parts.push({ type: 'text', text: textBeforeUrl }) try {
} const response = await ky.get(cleanUrl)
const cleanUrl = url.trim() if (
!response.ok ||
try { !response.headers.get('content-type')?.startsWith('image/')
const response = await ky.get(cleanUrl) ) {
if ( parts.push({ type: 'text', text: cleanUrl })
!response.ok || } else {
!response.headers.get('content-type')?.startsWith('image/') parts.push({
) { type: 'image',
parts.push({ type: 'text', text: cleanUrl }) image: shouldDownloadImages
} else { ? await response.arrayBuffer()
parts.push({ : url.trim(),
type: 'image', })
image: shouldDownloadImages }
? await response.arrayBuffer() } catch (err) {
: url.trim(), if (err instanceof HTTPError) {
}) console.log(err.response.status, await err.response.text())
} else {
console.error(err)
}
}
} }
} catch (err) { } else {
if (err instanceof HTTPError) { if (parts.at(-1)?.type === 'text') {
console.log(err.response.status, await err.response.text()) const lastText = parts.at(-1) as TextPart
parts = parts.slice(0, -1)
parts.push({ type: 'text', text: lastText.text + '\n\n' + part })
} else { } else {
console.error(err) parts.push({ type: 'text', text: part })
} }
} }
processedInput = processedInput.slice(
processedInput.indexOf(url) + url.length
)
}
if (processedInput.trim().length > 0) {
parts.push({ type: 'text', text: processedInput })
} }
return parts return parts

7
packages/bot-engine/blocks/logic/setVariable/executeSetVariable.ts
View File

@ -298,13 +298,6 @@ const getExpressionToEvaluate =
type: 'value', type: 'value',
value: [item], value: [item],
} }
if (isEmpty(item))
return {
type: 'value',
value: Array.isArray(variableValue)
? variableValue
: [variableValue],
}
if (!Array.isArray(variableValue)) if (!Array.isArray(variableValue))
return { type: 'value', value: [variableValue, item] } return { type: 'value', value: [variableValue, item] }
return { type: 'value', value: variableValue.concat(item) } return { type: 'value', value: variableValue.concat(item) }

70
packages/forge/blocks/openai/helpers/splitUserTextMessageIntoOpenAIBlocks.ts
View File

@ -4,49 +4,43 @@ import OpenAI from 'openai'
export const splitUserTextMessageIntoOpenAIBlocks = async ( export const splitUserTextMessageIntoOpenAIBlocks = async (
input: string input: string
): Promise<string | OpenAI.Chat.ChatCompletionContentPart[]> => { ): Promise<string | OpenAI.Chat.ChatCompletionContentPart[]> => {
const urlRegex = /(^|\n\n)(https?:\/\/[^\s]+)(\n\n|$)/g const splittedInput = input.split('\n\n')
const match = input.match(urlRegex)
if (!match) return input
let parts: OpenAI.Chat.ChatCompletionContentPart[] = [] let parts: OpenAI.Chat.ChatCompletionContentPart[] = []
let processedInput = input for (const part of splittedInput) {
if (part.startsWith('http') || part.startsWith('["http')) {
for (const url of match) { const urls = part.startsWith('[') ? JSON.parse(part) : [part]
const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url)) for (const url of urls) {
if (textBeforeUrl.trim().length > 0) { const cleanUrl = url.trim()
parts.push({ type: 'text', text: textBeforeUrl }) try {
} const response = await ky.get(cleanUrl)
const cleanUrl = url.trim() if (
!response.ok ||
try { !response.headers.get('content-type')?.startsWith('image/')
const response = await ky.get(cleanUrl) ) {
if ( parts.push({ type: 'text', text: cleanUrl })
!response.ok || } else {
!response.headers.get('content-type')?.startsWith('image/') parts.push({
) { type: 'image_url',
parts.push({ type: 'text', text: cleanUrl }) image_url: url.trim(),
} else { })
parts.push({ }
type: 'image_url', } catch (err) {
image_url: { if (err instanceof HTTPError) {
url: url.trim(), console.log(err.response.status, await err.response.text())
detail: 'auto', } else {
}, console.error(err)
}) }
}
} }
} catch (err) { } else {
if (err instanceof HTTPError) { if (parts.at(-1)?.type === 'text') {
console.log(err.response.status, await err.response.text()) const lastText = parts.at(-1) as OpenAI.ChatCompletionContentPartText
parts = parts.slice(0, -1)
parts.push({ type: 'text', text: lastText.text + '\n\n' + part })
} else { } else {
console.error(err) parts.push({ type: 'text', text: part })
} }
} }
processedInput = processedInput.slice(
processedInput.indexOf(url) + url.length
)
}
if (processedInput.trim().length > 0) {
parts.push({ type: 'text', text: processedInput })
} }
return parts return parts