SmartScraper

from scrapegraph_py import Client

client = Client(api_key="your-api-key")

response = client.smartscraper(
    website_url="https://scrapegraphai.com/",
    user_prompt="Extract info about the company"
)

import { smartScraper } from 'scrapegraph-js';

const apiKey = 'your-api-key';

const response = await smartScraper(apiKey, {
  website_url: 'https://scrapegraphai.com',
  user_prompt: 'What does the company do?',
});

if (response.status === 'error') {
  console.error(response.error);
} else {
  console.log(response.data);
}

curl -X 'POST' \
  'https://api.scrapegraphai.com/v1/smartscraper' \
  -H 'accept: application/json' \
  -H 'SGAI-APIKEY: your-api-key' \
  -H 'Content-Type: application/json' \
  -d '{
  "website_url": "https://example-shop.com/products/456",
  "user_prompt": "Extract product details including name, price, availability, specifications, and customer ratings."
}'

just-scrape smart-scraper https://scrapegraphai.com/ -p "Extract info about the company"

import { history } from 'scrapegraph-js';

const apiKey = 'your_api_key';

const response = await history(apiKey, {
  service: 'smartscraper',
  page: 1,
  page_size: 10,
});

if (response.status === 'success') {
  console.log(response.data.requests);
}

from pydantic import BaseModel, Field

class ArticleData(BaseModel):
    title: str = Field(description="Article title")
    author: str = Field(description="Author name")
    content: str = Field(description="Main article content")
    publish_date: str = Field(description="Publication date")

response = client.smartscraper(
    website_url="https://example.com/article",
    user_prompt="Extract the article information",
    output_schema=ArticleData
)

import { smartScraper } from 'scrapegraph-js';
import { z } from 'zod';

const apiKey = 'your_api_key';

const schema = z.object({
  title: z.string().describe('The title of the webpage'),
  description: z.string().describe('The description of the webpage'),
  summary: z.string().describe('A brief summary of the webpage'),
});

const response = await smartScraper(apiKey, {
  website_url: 'https://scrapegraphai.com/',
  user_prompt: 'What does the company do?',
  output_schema: schema,
});

if (response.status === 'success') {
  console.log(response.data.result);
}

import asyncio
from scrapegraph_py import AsyncClient
from pydantic import BaseModel, Field

# Define your schema
class WebpageSchema(BaseModel):
    title: str = Field(description="The title of the webpage")
    description: str = Field(description="The description of the webpage")
    summary: str = Field(description="A brief summary of the webpage")

async def main():
    # Initialize the async client
    async with AsyncClient(api_key="your-api-key") as client:
        # List of URLs to analyze
        urls = [
            "https://scrapegraphai.com/",
            "https://github.com/ScrapeGraphAI/Scrapegraph-ai",
        ]

        # Create scraping tasks for each URL
        tasks = [
            client.smartscraper(
                website_url=url,
                user_prompt="Summarize the main content",
                output_schema=WebpageSchema
            )
            for url in urls
        ]

        # Execute requests concurrently
        responses = await asyncio.gather(*tasks, return_exceptions=True)

        # Process results
        for i, response in enumerate(responses):
            if isinstance(response, Exception):
                print(f"Error for {urls[i]}: {response}")
            else:
                print(f"Result for {urls[i]}: {response['result']}")

# Run the async function
if __name__ == "__main__":
    asyncio.run(main())

import { smartScraper } from 'scrapegraph-js';
import { z } from 'zod';

const WebpageSchema = z.object({
  title: z.string().describe('The title of the webpage'),
  description: z.string().describe('The description of the webpage'),
  summary: z.string().describe('A brief summary of the webpage')
});

const apiKey = 'your-api-key';
const urls = [
  'https://scrapegraphai.com/',
  'https://github.com/ScrapeGraphAI/Scrapegraph-ai'
];

const results = await Promise.all(
  urls.map(url =>
    smartScraper(apiKey, {
      website_url: url,
      user_prompt: 'Summarize the main content',
      output_schema: WebpageSchema,
    })
  )
);

results.forEach((result, index) => {
  if (result.status === 'success') {
    console.log(`Results for ${urls[index]}:`, result.data.result);
  } else {
    console.error(`Error for ${urls[index]}:`, result.error);
  }
});

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
from pydantic import BaseModel
from typing import List

sgai_logger.set_logging(level="INFO")

# Define the output schema
class Company(BaseModel):
    name: str
    category: str
    location: str

class CompaniesResponse(BaseModel):
    companies: List[Company]

# Initialize the client with explicit API key
sgai_client = Client(api_key="sgai-api-key")

try:
    # SmartScraper request with infinite scroll
    response = sgai_client.smartscraper(
        website_url="https://www.ycombinator.com/companies?batch=Spring%202025",
        user_prompt="Extract all company names and their categories from the page",
        output_schema=CompaniesResponse,
        number_of_scrolls=10  # Scroll 10 times to load more companies
    )

    # Print the response
    print(f"Request ID: {response['request_id']}")
    
    # Parse and print the results in a structured way
    result = CompaniesResponse.model_validate(response['result'])
    print("\nExtracted Companies:")
    print("-" * 80)
    for company in result.companies:
        print(f"Name: {company.name}")
        print(f"Category: {company.category}")
        print(f"Location: {company.location}")
        print("-" * 80)

except Exception as e:
    print(f"An error occurred: {e}")

finally:
    sgai_client.close()

import { smartScraper } from 'scrapegraph-js';
import 'dotenv/config';

const apiKey = process.env.SGAI_APIKEY;

const response = await smartScraper(apiKey, {
  website_url: 'https://example.com/infinite-scroll-page',
  user_prompt: 'Extract all the posts from the feed',
  number_of_scrolls: 10,
});

if (response.status === 'success') {
  console.log('Extracted data from scrolled page:', response.data);
} else {
  console.error('Error:', response.error);
}