Handling pagination

from scrapegraph_py import ScrapeGraphAI, JsonFormatConfig

sgai = ScrapeGraphAI()

start = sgai.crawl.start(
    "https://example.com/products",
    formats=[JsonFormatConfig(prompt="Extract product names and prices")],
    max_depth=2,
    max_pages=50,
    include_patterns=["/products*"],
)

status = sgai.crawl.get(start.data.id)
print(f"{status.data.finished}/{status.data.total} - {status.data.status}")

import { ScrapeGraphAI } from "scrapegraph-js";

const sgai = ScrapeGraphAI();

const start = await sgai.crawl.start({
  url: "https://example.com/products",
  formats: [{ type: "json", prompt: "Extract product names and prices" }],
  maxDepth: 2,
  maxPages: 50,
  includePatterns: ["/products*"],
});

const status = await sgai.crawl.get(start.data.id);

from scrapegraph_py import ScrapeGraphAI

sgai = ScrapeGraphAI()

all_products = []
for page in range(1, 6):  # pages 1-5
    url = f"https://example.com/products?page={page}"
    res = sgai.extract(
        "Extract all product names and prices on this page",
        url=url,
    )
    if res.status == "success":
        all_products.extend(res.data.json_data.get("products", []))

print(f"Total products extracted: {len(all_products)}")

import { ScrapeGraphAI } from "scrapegraph-js";

const sgai = ScrapeGraphAI();
const allProducts = [];

for (let page = 1; page <= 5; page++) {
  const url = `https://example.com/products?page=${page}`;
  const res = await sgai.extract({
    url,
    prompt: "Extract all product names and prices on this page",
  });
  if (res.status === "success") {
    allProducts.push(...(res.data?.json?.products ?? []));
  }
}