Crawl4AITool

autogen.tools.experimental.Crawl4AITool

Crawl4AITool(llm_config=None, extraction_model=None, llm_strategy_kwargs=None)

Bases: Tool

Crawl a website and extract information using the crawl4ai library.

Parameters

Source code in autogen/tools/experimental/crawl4ai/crawl4ai.py

async def crawl4ai_helper(
    url: str,
    browser_cfg: Optional["BrowserConfig"] = None,
    crawl_config: Optional["CrawlerRunConfig"] = None,
) -> Any:
    async with AsyncWebCrawler(config=browser_cfg) as crawler:
        result = await crawler.arun(
            url=url,
            config=crawl_config,
        )
    if crawl_config is None:
        response = result.markdown
    else:
        response = result.extracted_content if result.success else result.error_message
    return response

async def crawl4ai_without_llm(
    url: Annotated[str, "The url to crawl and extract information from."],
) -> Any:
    return await crawl4ai_helper(url=url)

async def crawl4ai_with_llm(
    url: Annotated[str, "The url to crawl and extract information from."],
    instruction: Annotated[str, "The instruction to provide on how and what to extract."],
    llm_config: Annotated[Any, Depends(on(llm_config))],
    llm_strategy_kwargs: Annotated[dict[str, Any] | None, Depends(on(llm_strategy_kwargs))],
    extraction_model: Annotated[type[BaseModel] | None, Depends(on(extraction_model))],
) -> Any:
    browser_cfg = BrowserConfig(headless=True)
    crawl_config = Crawl4AITool._get_crawl_config(
        llm_config=llm_config,
        instruction=instruction,
        extraction_model=extraction_model,
        llm_strategy_kwargs=llm_strategy_kwargs,
    )
    return await crawl4ai_helper(url=url, browser_cfg=browser_cfg, crawl_config=crawl_config)

super().__init__(
    name="crawl4ai",
    description="Crawl a website and extract information.",
    func_or_tool=crawl4ai_without_llm if llm_config is None else crawl4ai_with_llm,
)

name property

name

description property

description

func property

func

tool_schema property

tool_schema

Get the schema for the tool.

function_schema property

function_schema

Get the schema for the function.

realtime_tool_schema property

realtime_tool_schema

Get the schema for the tool.

register_for_llm

register_for_llm(agent)

Registers the tool for use with a ConversableAgent's language model (LLM).

Parameters

Source code in autogen/tools/tool.py

def register_for_llm(self, agent: "ConversableAgent") -> None:
    """Registers the tool for use with a ConversableAgent's language model (LLM).
    This method registers the tool so that it can be invoked by the agent during
    interactions with the language model.
    Args:
        agent (ConversableAgent): The agent to which the tool will be registered.
    """

register_for_execution

register_for_execution(agent)

Registers the tool for direct execution by a ConversableAgent.

Parameters

Source code in autogen/tools/tool.py

def register_for_execution(self, agent: "ConversableAgent") -> None:
    """Registers the tool for direct execution by a ConversableAgent.
    This method registers the tool so that it can be executed by the agent,
    typically outside of the context of an LLM interaction.
    Args:
        agent (ConversableAgent): The agent to which the tool will be registered.
    """

register_tool

register_tool(agent)

Register a tool to be both proposed and executed by an agent.

Parameters

Source code in autogen/tools/tool.py

def register_tool(self, agent: "ConversableAgent") -> None:
    """Register a tool to be both proposed and executed by an agent.
    Equivalent to calling both `register_for_llm` and `register_for_execution` with the same agent.
    Note: This will not make the agent recommend and execute the call in the one step. If the agent
    recommends the tool, it will need to be the next agent to speak in order to execute the tool.
    Args:
        agent (ConversableAgent): The agent to which the tool will be registered.
    """