Query the OpenAI Vision API with an Image URL

Template

Function

Template

Function

FlowBridge

Easily query the OpenAI Vision API with an image URL and map the 'response' output variable to a variable you have set on your Voiceflow project.

Created By

FlowBridge

Function Code Snippet


export default async function main({ inputVars }) {
  const { openai_token, query, image_url, max_tokens } = inputVars

    if (!image_url || !openai_token || !query || !max_tokens) {
        return {
          next: {
            path: 'error'
          },
          trace: [
            {
              type: "debug",
              payload: {
                message: "Not all values were filled in for the Vision API function"
              }
            }
          ]
        }
    }

    let payload = {
      "model": "gpt-4-vision-preview",
      "messages": [
        {
          "role": "user",
          "content": [
            {
              "type": "text",
              "text": query
            },
            {
              "type": "image_url",
              "image_url": {
                "url": image_url
              }
            }
          ]
        }
      ],
      "max_tokens": max_tokens
    }
  
    const response = (
      await fetch('https://api.openai.com/v1/chat/completions', {
          method: 'POST',
          headers: {
              'Authorization': 'Bearer '+ openai_token,
              'Content-Type': 'application/json'
          },
          body: JSON.stringify(payload)
        },
        { parseType: 'json' }
      )
    ).json

  if(!response || !response?.choices[0]?.message?.content) {
    return {
      next: {
        path: 'error'
      },
      trace: [
        {
          type: "debug",
          payload: {
            message: "Something went wrong while querying the OpenAI Vision API"
          }
        },
        {
          type: "debug",
          payload: {
            message: "Open AI Response: "+ JSON.stringify(response)
          }
        }
      ]
    }
  }
  
  return {
    outputVars: {
      response: response.choices[0].message.content
    },
    next: {
      path: 'success'
    },
    trace: [
      {
        type: "debug",
        payload: {
          message: "Open AI Response: "+ JSON.stringify(response)
        }
      },
      {
        type: "debug",
        payload: {
          message: "Vision Message: "+ response.choices[0].message.content
        }
      }
    ]
  }
       
}