Transition from Firecrawl to ScrapeGraph v2

# Before (Firecrawl)
from firecrawl import Firecrawl
fc = Firecrawl(api_key="fc-...")

# After (ScrapeGraph v2)
from scrapegraph_py import ScrapeGraphAI
# reads SGAI_API_KEY from env, or pass explicitly: ScrapeGraphAI(api_key="...")
sgai = ScrapeGraphAI()

// Before (Firecrawl)
import Firecrawl from "@mendable/firecrawl-js";
const fc = new Firecrawl({ apiKey: "fc-..." });

// After (ScrapeGraph v2)
import { ScrapeGraphAI } from "scrapegraph-js";
// reads SGAI_API_KEY from env, or pass explicitly: ScrapeGraphAI({ apiKey: "..." })
const sgai = new ScrapeGraphAI();

# Before (Firecrawl)
doc = fc.scrape("https://example.com", formats=["markdown"])
print(doc.markdown)

# After (ScrapeGraph v2 — scrapegraph-py ≥ 2.1.0)
from scrapegraph_py import MarkdownFormatConfig

res = sgai.scrape(
    "https://example.com",
    formats=[MarkdownFormatConfig()],
)
if res.status == "success":
    print(res.data.results["markdown"]["data"][0])

// Before (Firecrawl)
const doc = await fc.scrape("https://example.com", { formats: ["markdown"] });
console.log(doc.markdown);

// After (ScrapeGraph v2)
const res = await sgai.scrape({
  url: "https://example.com",
  formats: [{ type: "markdown" }],
});
if (res.status === "success") {
  console.log(res.data?.results.markdown?.data?.[0]);
}

# Before (Firecrawl)
doc = fc.scrape("https://example.com", formats=["markdown", "html", "links", "screenshot"])
print(doc.markdown, doc.html, doc.links, doc.screenshot)

# After (ScrapeGraph v2)
from scrapegraph_py import (
    MarkdownFormatConfig, HtmlFormatConfig,
    LinksFormatConfig, ScreenshotFormatConfig,
)

res = sgai.scrape(
    "https://example.com",
    formats=[
        MarkdownFormatConfig(),
        HtmlFormatConfig(mode="reader"),
        LinksFormatConfig(),
        ScreenshotFormatConfig(full_page=True, width=1440, height=900),
    ],
)
results = res.data.results
print(results["markdown"]["data"][0])
print(results["html"]["data"][0])
print(results["links"]["data"])
print(results["screenshot"]["data"][0])  # base64 PNG

// After (ScrapeGraph v2)
const res = await sgai.scrape({
  url: "https://example.com",
  formats: [
    { type: "markdown" },
    { type: "html", mode: "reader" },
    { type: "links" },
    { type: "screenshot", fullPage: true, width: 1440, height: 900 },
  ],
});
const r = res.data?.results;
console.log(r?.markdown?.data?.[0]);
console.log(r?.screenshot?.data?.[0]); // base64 PNG

# Before (Firecrawl — actions array)
doc = fc.scrape(
    "https://example.com",
    formats=["markdown"],
    actions=[
        {"type": "wait", "milliseconds": 2000},
        {"type": "scroll", "direction": "down"},
        {"type": "scroll", "direction": "down"},
    ],
)

# After (ScrapeGraph v2 — declarative fetchConfig)
from scrapegraph_py import MarkdownFormatConfig, FetchConfig

res = sgai.scrape(
    "https://example.com",
    formats=[MarkdownFormatConfig()],
    fetch_config=FetchConfig(
        mode="js",         # render JavaScript
        stealth=True,      # rotate residential proxy
        wait=2000,         # ms after navigation
        scrolls=2,         # programmatic scroll ticks
        country="US",
    ),
)

// After (ScrapeGraph v2)
const res = await sgai.scrape({
  url: "https://example.com",
  formats: [{ type: "markdown" }],
  fetchConfig: {
    mode: "js",
    stealth: true,
    wait: 2000,
    scrolls: 2,
    country: "US",
  },
});

# Before (Firecrawl)
result = fc.extract(
    urls=["https://example.com"],
    prompt="Extract the main heading",
    schema={"type": "object", "properties": {"title": {"type": "string"}}},
)

# After (ScrapeGraph v2 — scrapegraph-py ≥ 2.1.0)
res = sgai.extract(
    "Extract the main heading",
    url="https://example.com",
    schema={"type": "object", "properties": {"title": {"type": "string"}}},
)
if res.status == "success":
    print(res.data.json_data)

// Before (Firecrawl)
const result = await fc.extract({
  urls: ["https://example.com"],
  prompt: "Extract the main heading",
  schema: { type: "object", properties: { title: { type: "string" } } },
});

// After (ScrapeGraph v2)
const res = await sgai.extract({
  url: "https://example.com",
  prompt: "Extract the main heading",
  schema: { type: "object", properties: { title: { type: "string" } } },
});
if (res.status === "success") {
  console.log(res.data?.json);
}

# Before (Firecrawl)
hits = fc.search(query="best programming languages 2026", limit=5)

# After (ScrapeGraph v2 — scrapegraph-py ≥ 2.1.0)
res = sgai.search(
    "best programming languages 2026",
    num_results=5,
)
if res.status == "success":
    for r in res.data.results:
        print(r.title, "-", r.url)

// Before (Firecrawl)
const hits = await fc.search({ query: "best programming languages 2026", limit: 5 });

// After (ScrapeGraph v2)
const res = await sgai.search({
  query: "best programming languages 2026",
  numResults: 5,
});
if (res.status === "success") {
  for (const r of res.data?.results ?? []) console.log(r.title, "-", r.url);
}

# Before (Firecrawl — blocking)
job = fc.crawl("https://example.com", limit=50)

# Or non-blocking:
started = fc.start_crawl("https://example.com", limit=50)
status = fc.get_crawl_status(started.id)

# After (ScrapeGraph v2 — scrapegraph-py ≥ 2.1.0)
start = sgai.crawl.start(
    "https://example.com",
    max_depth=2,
    max_pages=50,
    include_patterns=["/blog/*"],
    exclude_patterns=["/admin/*"],
)
status = sgai.crawl.get(start.data.id)
print(status.data.status, status.data.finished, "/", status.data.total)

// Before (Firecrawl)
const job = await fc.crawl("https://example.com", { limit: 50 });
// Or non-blocking:
const started = await fc.startCrawl("https://example.com", { limit: 50 });
const status = await fc.getCrawlStatus(started.id);

// After (ScrapeGraph v2)
const start = await sgai.crawl.start({
  url: "https://example.com",
  maxDepth: 2,
  maxPages: 50,
  includePatterns: ["/blog/*"],
  excludePatterns: ["/admin/*"],
});
const status = await sgai.crawl.get(start.data.id);

# Before (Firecrawl — add changeTracking to formats)
doc = fc.scrape(
    "https://example.com",
    formats=["markdown", {"type": "changeTracking", "modes": ["git-diff"], "tag": "hourly"}],
)

# After (ScrapeGraph v2 — scheduled monitor, scrapegraph-py ≥ 2.1.0)
from scrapegraph_py import MarkdownFormatConfig

res = sgai.monitor.create(
    "https://example.com",
    "*/30 * * * *",                 # cron expression (positional)
    name="Homepage watch",
    formats=[MarkdownFormatConfig()],
    webhook_url="https://your-app.example.com/hooks/sgai",
)
cron_id = res.data.cron_id

// Before (Firecrawl)
const doc = await fc.scrape("https://example.com", {
  formats: ["markdown", { type: "changeTracking", modes: ["git-diff"], tag: "hourly" }],
});

// After (ScrapeGraph v2)
const res = await sgai.monitor.create({
  url: "https://example.com",
  name: "Homepage watch",
  interval: "*/30 * * * *",
  formats: [{ type: "markdown" }],
  webhookUrl: "https://your-app.example.com/hooks/sgai",
});
const cronId = res.data?.cronId;

import asyncio
from scrapegraph_py import AsyncScrapeGraphAI, MarkdownFormatConfig

async def fetch_many(urls):
    async with AsyncScrapeGraphAI() as sgai:
        return await asyncio.gather(*[
            sgai.scrape(u, formats=[MarkdownFormatConfig()]) for u in urls
        ])

results = asyncio.run(fetch_many([
    "https://example.com",
    "https://example.org",
]))

// The default `ScrapeGraphAI` client is already promise-based.
const urls = ["https://example.com", "https://example.org"];
const results = await Promise.all(urls.map((url) =>
  sgai.scrape({ url, formats: [{ type: "markdown" }] })
));