SmartScraper

from scrapegraph_py import Client

client = Client(api_key="your-api-key")

response = client.smartscraper(
    website_url="https://scrapegraphai.com/",
    user_prompt="Extract info about the company"
)

import { getSmartScraperRequest } from 'scrapegraph-js';

const apiKey = 'your_api_key';
const requestId = 'ID_of_previous_request';

try {
  const requestInfo = await getSmartScraperRequest(apiKey, requestId);
  console.log(requestInfo);
} catch (error) {
  console.error(error);
}

from pydantic import BaseModel, Field

class ArticleData(BaseModel):
    title: str = Field(description="Article title")
    author: str = Field(description="Author name")
    content: str = Field(description="Main article content")
    publish_date: str = Field(description="Publication date")

response = client.smartscraper(
    website_url="https://example.com/article",
    user_prompt="Extract the article information",
    output_schema=ArticleData
)

import asyncio
from scrapegraph_py import AsyncClient
from pydantic import BaseModel, Field

# Define your schema
class WebpageSchema(BaseModel):
    title: str = Field(description="The title of the webpage")
    description: str = Field(description="The description of the webpage")
    summary: str = Field(description="A brief summary of the webpage")

async def main():
    # Initialize the async client
    async with AsyncClient(api_key="your-api-key") as client:
        # List of URLs to analyze
        urls = [
            "https://scrapegraphai.com/",
            "https://github.com/ScrapeGraphAI/Scrapegraph-ai",
        ]

        # Create scraping tasks for each URL
        tasks = [
            client.smartscraper(
                website_url=url,
                user_prompt="Summarize the main content",
                output_schema=WebpageSchema
            )
            for url in urls
        ]

        # Execute requests concurrently
        responses = await asyncio.gather(*tasks, return_exceptions=True)

        # Process results
        for i, response in enumerate(responses):
            if isinstance(response, Exception):
                print(f"Error for {urls[i]}: {response}")
            else:
                print(f"Result for {urls[i]}: {response['result']}")

# Run the async function
if __name__ == "__main__":
    asyncio.run(main())

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
from pydantic import BaseModel
from typing import List

sgai_logger.set_logging(level="INFO")

# Define the output schema
class Company(BaseModel):
    name: str
    category: str
    location: str

class CompaniesResponse(BaseModel):
    companies: List[Company]

# Initialize the client with explicit API key
sgai_client = Client(api_key="sgai-api-key")

try:
    # SmartScraper request with infinite scroll
    response = sgai_client.smartscraper(
        website_url="https://www.ycombinator.com/companies?batch=Spring%202025",
        user_prompt="Extract all company names and their categories from the page",
        output_schema=CompaniesResponse,
        number_of_scrolls=10  # Scroll 10 times to load more companies
    )

    # Print the response
    print(f"Request ID: {response['request_id']}")
    
    # Parse and print the results in a structured way
    result = CompaniesResponse.model_validate(response['result'])
    print("\nExtracted Companies:")
    print("-" * 80)
    for company in result.companies:
        print(f"Name: {company.name}")
        print(f"Category: {company.category}")
        print(f"Location: {company.location}")
        print("-" * 80)

except Exception as e:
    print(f"An error occurred: {e}")

finally:
    sgai_client.close()