CrewAI is a framework for orchestrating role-playing AI agents. With the Scrapegraph CrewAI integration, you can easily incorporate web scraping capabilities into your agent workflows.
The ScrapegraphScrapeTool provides web scraping capabilities to your CrewAI agents:
from crewai import Agent, Crew, Taskfrom crewai_tools import ScrapegraphScrapeToolfrom dotenv import load_dotenv# Initialize the tooltool = ScrapegraphScrapeTool()# Create an agent with the toolagent = Agent( role="Web Researcher", goal="Research and extract accurate information from websites", backstory="You are an expert web researcher with experience in extracting and analyzing information from various websites.", tools=[tool],)
from crewai import Agent, Crew, Taskfrom crewai_tools import ScrapegraphScrapeToolfrom dotenv import load_dotenv# Load environment variablesload_dotenv()# Initialize the Scrapegraph tooltool = ScrapegraphScrapeTool()# Create an agent with the Scrapegraph toolagent = Agent( role="Web Researcher", goal="Research and extract accurate information from websites", backstory="You are an expert web researcher with experience in extracting and analyzing information from various websites.", tools=[tool],)# Define a task for the agenttask = Task( name="scraping task", description="Visit the website https://scrapegraphai.com and extract detailed information about the founders, including their names, roles, and any relevant background information.", expected_output="A file with the information extracted from the website.", agent=agent,)# Create a crew with the agent and taskcrew = Crew( agents=[agent], tasks=[task],)# Execute the taskresult = crew.kickoff()
