From 3b87801c922bcdc53ddbab2e5435f885b717f692 Mon Sep 17 00:00:00 2001 From: Baptiste Arnaud Date: Wed, 4 Sep 2024 14:07:32 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20(ai)=20Enable=20multi=20urls=20f?= =?UTF-8?q?or=20vision?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1764 --- packages/ai/splitUserTextMessageIntoBlocks.ts | 71 +++++++++---------- .../logic/setVariable/executeSetVariable.ts | 7 -- .../splitUserTextMessageIntoOpenAIBlocks.ts | 70 +++++++++--------- 3 files changed, 66 insertions(+), 82 deletions(-) diff --git a/packages/ai/splitUserTextMessageIntoBlocks.ts b/packages/ai/splitUserTextMessageIntoBlocks.ts index b7dd1f35f..700eced53 100644 --- a/packages/ai/splitUserTextMessageIntoBlocks.ts +++ b/packages/ai/splitUserTextMessageIntoBlocks.ts @@ -9,48 +9,45 @@ export const splitUserTextMessageIntoBlocks = async ({ input, shouldDownloadImages, }: Props): Promise => { - const urlRegex = /(^|\n\n)(https?:\/\/.+)(\n\n|$)/g - const match = input.match(urlRegex) - if (!match) return input + const splittedInput = input.split('\n\n') let parts: (TextPart | ImagePart)[] = [] - let processedInput = input - - for (const url of match) { - const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url)) - if (textBeforeUrl.trim().length > 0) { - parts.push({ type: 'text', text: textBeforeUrl }) - } - const cleanUrl = url.trim() - - try { - const response = await ky.get(cleanUrl) - if ( - !response.ok || - !response.headers.get('content-type')?.startsWith('image/') - ) { - parts.push({ type: 'text', text: cleanUrl }) - } else { - parts.push({ - type: 'image', - image: shouldDownloadImages - ? await response.arrayBuffer() - : url.trim(), - }) + for (const part of splittedInput) { + if (part.startsWith('http') || part.startsWith('["http')) { + const urls = part.startsWith('[') ? JSON.parse(part) : [part] + for (const url of urls) { + const cleanUrl = url.trim() + try { + const response = await ky.get(cleanUrl) + if ( + !response.ok || + !response.headers.get('content-type')?.startsWith('image/') + ) { + parts.push({ type: 'text', text: cleanUrl }) + } else { + parts.push({ + type: 'image', + image: shouldDownloadImages + ? await response.arrayBuffer() + : url.trim(), + }) + } + } catch (err) { + if (err instanceof HTTPError) { + console.log(err.response.status, await err.response.text()) + } else { + console.error(err) + } + } } - } catch (err) { - if (err instanceof HTTPError) { - console.log(err.response.status, await err.response.text()) + } else { + if (parts.at(-1)?.type === 'text') { + const lastText = parts.at(-1) as TextPart + parts = parts.slice(0, -1) + parts.push({ type: 'text', text: lastText.text + '\n\n' + part }) } else { - console.error(err) + parts.push({ type: 'text', text: part }) } } - processedInput = processedInput.slice( - processedInput.indexOf(url) + url.length - ) - } - - if (processedInput.trim().length > 0) { - parts.push({ type: 'text', text: processedInput }) } return parts diff --git a/packages/bot-engine/blocks/logic/setVariable/executeSetVariable.ts b/packages/bot-engine/blocks/logic/setVariable/executeSetVariable.ts index e06149e1c..8e96e376f 100644 --- a/packages/bot-engine/blocks/logic/setVariable/executeSetVariable.ts +++ b/packages/bot-engine/blocks/logic/setVariable/executeSetVariable.ts @@ -298,13 +298,6 @@ const getExpressionToEvaluate = type: 'value', value: [item], } - if (isEmpty(item)) - return { - type: 'value', - value: Array.isArray(variableValue) - ? variableValue - : [variableValue], - } if (!Array.isArray(variableValue)) return { type: 'value', value: [variableValue, item] } return { type: 'value', value: variableValue.concat(item) } diff --git a/packages/forge/blocks/openai/helpers/splitUserTextMessageIntoOpenAIBlocks.ts b/packages/forge/blocks/openai/helpers/splitUserTextMessageIntoOpenAIBlocks.ts index c714336ad..784b84190 100644 --- a/packages/forge/blocks/openai/helpers/splitUserTextMessageIntoOpenAIBlocks.ts +++ b/packages/forge/blocks/openai/helpers/splitUserTextMessageIntoOpenAIBlocks.ts @@ -4,49 +4,43 @@ import OpenAI from 'openai' export const splitUserTextMessageIntoOpenAIBlocks = async ( input: string ): Promise => { - const urlRegex = /(^|\n\n)(https?:\/\/[^\s]+)(\n\n|$)/g - const match = input.match(urlRegex) - if (!match) return input + const splittedInput = input.split('\n\n') let parts: OpenAI.Chat.ChatCompletionContentPart[] = [] - let processedInput = input - - for (const url of match) { - const textBeforeUrl = processedInput.slice(0, processedInput.indexOf(url)) - if (textBeforeUrl.trim().length > 0) { - parts.push({ type: 'text', text: textBeforeUrl }) - } - const cleanUrl = url.trim() - - try { - const response = await ky.get(cleanUrl) - if ( - !response.ok || - !response.headers.get('content-type')?.startsWith('image/') - ) { - parts.push({ type: 'text', text: cleanUrl }) - } else { - parts.push({ - type: 'image_url', - image_url: { - url: url.trim(), - detail: 'auto', - }, - }) + for (const part of splittedInput) { + if (part.startsWith('http') || part.startsWith('["http')) { + const urls = part.startsWith('[') ? JSON.parse(part) : [part] + for (const url of urls) { + const cleanUrl = url.trim() + try { + const response = await ky.get(cleanUrl) + if ( + !response.ok || + !response.headers.get('content-type')?.startsWith('image/') + ) { + parts.push({ type: 'text', text: cleanUrl }) + } else { + parts.push({ + type: 'image_url', + image_url: url.trim(), + }) + } + } catch (err) { + if (err instanceof HTTPError) { + console.log(err.response.status, await err.response.text()) + } else { + console.error(err) + } + } } - } catch (err) { - if (err instanceof HTTPError) { - console.log(err.response.status, await err.response.text()) + } else { + if (parts.at(-1)?.type === 'text') { + const lastText = parts.at(-1) as OpenAI.ChatCompletionContentPartText + parts = parts.slice(0, -1) + parts.push({ type: 'text', text: lastText.text + '\n\n' + part }) } else { - console.error(err) + parts.push({ type: 'text', text: part }) } } - processedInput = processedInput.slice( - processedInput.indexOf(url) + url.length - ) - } - - if (processedInput.trim().length > 0) { - parts.push({ type: 'text', text: processedInput }) } return parts