/*
 * Purpose of This File:
 * This file is required for Quarto to render both HTML and MDX files in the `website` folder.
 * These rendered files are essential for Mintlify to function correctly.
 *
 * Context:
 * - The `quarto render` command behaves differently when the `docusaurus.config.js` file is
 *   present in the `website` folder.
 * - Without this file, Quarto does not render MDX files, leading to failures
 *   when running the `process_notebook.py` script.
 * - As a result, this file is retained temporarily until a better solution is identified.
 *
 * Options Tried (and Issues Faced):
 * 1. Adding a `_quarto.yml` file in the `website` folder with the following configurations:
 *
 *    **Minimal Configuration:**
 *    ```
 *    project:
 *      type: docusaurus
 *
 *    format:
 *      html: default
 *      docusaurus-md: default
 *    ```
 *
 *    **Full Configuration from Quarto Repository:**
 *    The configuration from [Quarto's docusaurus extension](https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/extensions/quarto/docusaurus/_extension.yml).
 *
 *    - Both configurations instruct Quarto to render HTML and MDX files.
 *    - However, in both cases, Mintlify fails to generate docs, raising the following error:
 *      > Warning: data for page "/[[...slug]]" (path "/docs/Home") is 56.1 MB which exceeds
 *      > the threshold of 21 MB. This amount of data can reduce performance.
 *    - More info: https://nextjs.org/docs/messages/large-page-data
 */


// Load jQuery if not present
if (typeof jQuery === "undefined") {
  const jqueryScript = document.createElement("script");
  jqueryScript.src =
    "https://cdnjs.cloudflare.com/ajax/libs/jquery/3.7.1/jquery.min.js";
  document.head.appendChild(jqueryScript);

  jqueryScript.onload = function () {
    // Load Chosen after jQuery
    const chosenScript = document.createElement("script");
    chosenScript.src =
      "https://cdnjs.cloudflare.com/ajax/libs/chosen/1.8.7/chosen.jquery.min.js";
    document.head.appendChild(chosenScript);

    // Add Chosen CSS
    const chosenStyles = document.createElement("link");
    chosenStyles.rel = "stylesheet";
    chosenStyles.href =
      "https://cdnjs.cloudflare.com/ajax/libs/chosen/1.8.7/chosen.min.css";
    document.head.appendChild(chosenStyles);

    // Initialize Chosen when everything is loaded
    chosenScript.onload = function () {
      initializeGallerySelect();
    };
  };
}

function getTagsFromURL() {
  const searchParams = new URLSearchParams(window.location.search);
  const tags = searchParams.get("tags");
  return tags ? tags.split(",") : [];
}

// Function to initialize Chosen on gallery select
function initializeGallerySelect() {
  // Add flag to check if already initialized
  const selectElement = $(".examples-gallery-container .tag-filter");
  if (selectElement.length && !selectElement.data("chosen-initialized")) {
    selectElement
      .chosen({
        width: "100%",
        placeholder_text_multiple: "Filter by tags",
      })
      .data("chosen-initialized", true);

    // Get tags from URL and update chosen
    const urlTags = getTagsFromURL();
    if (urlTags.length > 0) {
      selectElement.val(urlTags);
      selectElement.trigger("chosen:updated");

      // Trigger the gallery:tagChange event with URL tags
      const customEvent = new CustomEvent("gallery:tagChange", {
        detail: urlTags,
      });
      document.dispatchEvent(customEvent);
    }

    // Set up change handler
    selectElement.on("change", function (evt, params) {
      const selectedValues = $(this).val() || [];
      const customEvent = new CustomEvent("gallery:tagChange", {
        detail: selectedValues,
      });
      document.dispatchEvent(customEvent);
    });
  }
}

// Debounce function
function debounce(func, wait) {
  let timeout;
  return function executedFunction(...args) {
    const later = () => {
      clearTimeout(timeout);
      func(...args);
    };
    clearTimeout(timeout);
    timeout = setTimeout(later, wait);
  };
}

// Debounced version of initialize
const debouncedInitialize = debounce(initializeGallerySelect, 100);

// Watch for URL changes using MutationObserver
const observer = new MutationObserver((mutations) => {
  if (window.location.pathname.includes("/use-cases")) {
    debouncedInitialize();
  }
});

observer.observe(document.body, {
  childList: true,
  subtree: true,
  attributes: false,
  characterData: false,
});

// Initialize on page load
document.addEventListener("DOMContentLoaded", function () {
  if (window.jQuery && window.jQuery.fn.chosen) {
    initializeGallerySelect();
  }
});


get_image_data

autogen.agentchat.contrib.capabilities.vision_capability.get_image_data

Key Features

Quick Start

Community Gallery

Contributing to AG2

Setting up Development Environment

Documentation

File a Bug Report

Guidance for Maintainers

Pre-commit

Tests

Research

Frequently Asked Questions

Agent Monitoring and Debugging with AgentOps

Azure Cosmos DB

Composio

Databricks

Llamaindex

Mem0:Long-Term Memory and Personalization for Agents

MemGPT

Microsoft Fabric

Ollama

PGVector

Portkey Integration

Promptflow

Home

User Guide

API References

Use Cases

User Stories

Contribute

FAQs

Ecosystem

Blog

Installing AG2

ConversableAgent

Human in the loop

Tools

Code Execution

Tools with Secrets

Conversation Patterns Deep-dive

LLM Config Deep-dive

OpenAI

Amazon Bedrock

Anthropic

Cerebras

Cohere

DeepSeek

Google Gemini

Google Vertex AI

Groq

LiteLLM with watsonx

LM Studio

Mistral AI

Together AI

vLLM

Overview

CaptainAgent

Communication Agents

DocAgent

ReasoningAgent

WebSurferAgent

Customer Service

Game Design

Travel Planning

All Notebooks

Use conversational agents to solve a set of tasks with a sequence of chats.

Solving Multiple Tasks in a Sequence of Chats

Use conversational agents to solve a set of tasks with a sequence of async chats.

Solving Multiple Tasks in a Sequence of Async Chats

Explore the use of AG2's RetrieveChat for tasks like code generation from docstrings, answering complex questions with human feedback, and exploiting features like Update Context, custom prompts, and few-shot learning.

Using RetrieveChat for Retrieve Augmented Code Generation and Question Answering

Automated continual learning from new data.

Interactive LLM Agent Dealing with Data Stream

Agent Chat with Async Human Inputs

Using RetrieveChat Powered by MongoDB Atlas for Retrieve Augmented Code Generation and Question Answering

Auto Generated Agent Chat: Task Solving with Langchain Provided Tools as Functions

Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes.

Task Solving with Code Generation, Execution and Debugging

Provide capabilities of runtime logging for debugging and performance analysis.

Runtime Logging with AG2

Multimodal agent chat with DALL-E and GPT-4v.

Agent Chat with Multimodal Models: DALLE  and GPT-4V

This notebook demonstrates the usage of QdrantRetrieveUserProxyAgent for RAG.

Using RetrieveChat with Qdrant for Retrieve Augmented Code Generation and Question Answering

Enhanced Swarm Orchestration with AG2

Use tools to extract and translate the transcript of a video file.

Translating Video audio using Whisper and GPT-3.5-turbo

Use AgentOps to simplify the development process and monitor your agents in production.

Agent Tracking with AgentOps

Explore a group chat example using agents such as a coder and visualization agent.

Group Chat with Coder and Visualization Critic

Use ReasoningAgent for o1 style reasoning in Agentic workflows with LLMs using AG2

ReasoningAgent - Advanced LLM Reasoning with Multiple Search Strategies

DeepResearchAgent

Tools with Dependency Injection

A Uniform interface to call different LLMs

Natural language text to SQL query using the Spider text-to-SQL benchmark.

SQL Agent for Spider text-to-SQL benchmark

DeepSeek: Adding Browsing Capabilities to AG2

Introducing CaptainAgent, a powerful agent that can manage and orchestrate other agents and tools to solve complex tasks.

Adding Browsing Capabilities to AG2

Teach the agent news skills using natural language.

Auto Generated Agent Chat: Teaching AI New Skills via Natural Language Interaction

Equip your agent with functions that can efficiently implement features into your software application.

Writing a software application using function calls

Function Inception: Enable AG2 agents to update/remove functions during conversations.

Auto Generated Agent Chat: Function Inception

Agentic RAG workflow on tabular data from a PDF file

RealtimeAgent in a Swarm Orchestration

Using FalkorGraphRagCapability with agents for GraphRAG Question & Answering

Making OpenAI Assistants Teachable

Cross-Framework LLM Tool Integration with AG2

RealtimeAgent with local websocket connection

Using RetrieveChat Powered by PGVector for Retrieve Augmented Code Generation and Question Answering

OpenAI Assistants in AG2

Solve complex tasks with a chat nested as inner monologue.

Solving Complex Tasks with Nested Chats

This Jupyter Notebook showcases the integration of the Code Interpreter tool which executes Python code dynamically within applications.

Auto Generated Agent Chat: GPTAssistant with Code Interpreter

Introduce group chat with customized speaker selection method.

Group Chat with Customized Speaker Selection Method

OpenAI offers a functionality for defining a structure of the messages generated by LLMs, AG2 enables this functionality by propagating response_format passed to your agents to the underlying client.

Structured output

Agent Chat with custom model loading

Automatically build multi-agent system with AgentBuilder

AutoBuild

Explore the demonstration of the Finite State Machine implementation, which allows the user to input speaker transition constraints.

FSM - User can input speaker transition constraints

Neo4j GraphRAG utilises a knowledge graph and can be added as a capability to agents.

Using Neo4j's graph database with AG2 agents for Question & Answering

Use Databricks DBRX and Foundation Model APIs to build AG2 applications backed by open-source LLMs.

Use AG2 in Databricks with DBRX

Language Agent Tree Search

FalkorDB GraphRAG utilises a knowledge graph and can be added as a capability to agents. Together with a swarm orchestration of agents is highly effective at providing a RAG capability.

Using a local Telemetry server to monitor a GraphRAG agent

Explore the demonstration of the SocietyOfMindAgent in the AG2 library, which runs a group chat as an internal monologue, but appears to the external world as a single agent, offering a structured way to manage complex interactions among multiple agents and handle issues such as extracting responses from complex dialogues and dealing with context window constraints.

SocietyOfMindAgent

Perform research using a group chat with a number of specialized agents.

Perform Research with Multi-Agent Group Chat

Explore the utilization of large language models in automated group chat scenarios, where agents perform tasks collectively, demonstrating how they can be configured, interact with each other, and retrieve specific information from external resources.

Group Chat

StateFlow: Build workflows through state-oriented actions.

StateFlow: Build Workflows through State-Oriented Actions

Agent Chat with Multimodal Models: LLaVA

Trip planning with a FalkorDB GraphRAG agent using a Swarm

Involve multiple human users via function calls and nested chat.

Auto Generated Agent Chat: Collaborative Task Solving with Multiple Agents and Human Users

Agent with memory using Mem0

Preprocessing chat history with `TransformMessages`

Preprocessing Chat History with `TransformMessages`

AgentOptimizer is able to prompt LLMs to iteratively optimize function/skills of AutoGen agents according to the historical conversation and performance.

AgentOptimizer: An Agentic Way to Train Your LLM Agent

Using RetrieveChat Powered by Couchbase Capella for Retrieve Augmented Code Generation and Question Answering

Use OpenLIT to easily monitor AI agents in production with OpenTelemetry.

Agent Observability with OpenLIT

Learn how to persist memories across chat sessions using the Teachability capability

Chatting with a teachable agent

LLM-backed agents playing chess with each other using nested chats.

Conversational Chess using non-OpenAI clients

Solve complex tasks with one or more sequence chats nested as inner monologue.

Solving Complex Tasks with A Sequence of Nested Chats

Config loader utility functions

Learn how to register function calls using AssistantAgent and UserProxyAgent.

Currency Calculator: Task Solving with Provided Tools as Functions

This Jupyter Notebook demonstrates how to leverage OSS Insight (Open Source Software Insight) for advanced GitHub data analysis by defining `Function calls` in AG2 for the OpenAI Assistant.

Chat with OpenAI Assistant using function call in AG2: OSS Insights for Advanced GitHub Data Analysis

Scrapping web pages and summarizing the content using agents with tools.

Web Scraping using Apify Tools

Auto Generated Agent Chat: Collaborative Task Solving with Coding and Planning Agent

Query documents and web pages with DocAgent

RAG with DocAgent

AgentEval: a multi-agent system for assessing utility of LLM-powered applications

Demonstrating the `AgentEval` framework using the task of solving math problems as an example

RealtimeAgent with WebRTC connection

Automatically build multi-agent system from agent library

Automatically Build Multi-agent System from Agent Library

Integrate llamaindex agents with Autogen.

Groupchat with Llamaindex agents

Using Guidance with AG2

RealtimeAgent in a Swarm Orchestration using WebRTC

Cross-Framework LLM Tool for CaptainAgent

Websockets facilitate real-time, bidirectional communication between web clients and servers, enhancing the responsiveness and interactivity of AG2-powered applications.

Websockets: Streaming input and output using websockets

Auto Generated Agent Chat: Solving Tasks Requiring Web Info

Code generation, execution, debugging and human feedback.

Auto Generated Agent Chat: Task Solving with Code Generation, Execution, Debugging & Human Feedback

This notebook demonstrates the use of Assistant Agents in conjunction with Azure Cognitive Search and Azure Identity

Assistants with Azure Cognitive Search and Azure Identity

(Legacy) Implement Swarm-style orchestration with GroupChat

Register function calls using AssistantAgent and UserProxyAgent to execute python or shell code in customized ways. Demonstrating two ways of registering functions.

Auto Generated Agent Chat: Task Solving with Provided Tools as Functions

Auto Generated Agent Chat: Using MathChat to Solve Math Problems

Optimizing Small, Local LLMs in Multi-Agent RAG Workflows using IBM Granite, Document Retrieval, Web Search, and Ollama

Small, Local Model (IBM Granite) Multi-Agent RAG

Implement and manage a multi-agent chat system using AG2, where AI assistants retrieve information, generate code, and interact collaboratively to solve complex tasks, especially in areas not covered by their training data.

Name	Description
`*args`
`**kwargs`

API Reference

​get_image_data

get_image_data