OpenAI

npm install scrapegraph-js openai

SGAI_APIKEY=your_scrapegraph_key
OPENAI_API_KEY=your_openai_key

import { extract } from 'scrapegraph-js';
import OpenAI from 'openai';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

const result = await extract(process.env.SGAI_APIKEY!, {
    url: 'https://scrapegraphai.com',
    prompt: 'Extract all content from this page',
});

const data = result.data?.json;
console.log('Scraped content length:', JSON.stringify(data).length);

const completion = await openai.chat.completions.create({
    model: 'gpt-4o-mini',
    messages: [
        { role: 'user', content: `Summarize in 100 words: ${JSON.stringify(data)}` }
    ]
});

console.log('Response:', completion.choices[0].message.content);

import { extract } from 'scrapegraph-js';
import OpenAI from 'openai';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

console.log("Sending user message to OpenAI and requesting tool use if necessary...");
const response = await openai.chat.completions.create({
    model: 'gpt-4o-mini',
    messages: [{
        role: 'user',
        content: 'What is ScrapeGraphAI? Check scrapegraphai.com'
    }],
    tools: [{
        type: 'function',
        function: {
            name: 'scrape_website',
            description: 'Scrape and extract structured data from a website URL',
            parameters: {
                type: 'object',
                properties: {
                    url: { type: 'string', description: 'The URL to scrape' }
                },
                required: ['url']
            }
        }
    }]
});

const toolCall = response.choices[0].message.tool_calls?.[0];

if (toolCall) {
    const { url } = JSON.parse(toolCall.function.arguments);
    console.log(`Calling tool: ${toolCall.function.name} | URL: ${url}`);

    const result = await extract(process.env.SGAI_APIKEY!, {
        url,
        prompt: 'Extract all content from this page',
    });

    const data = result.data?.json;
    console.log(`Scraped content preview: ${JSON.stringify(data)?.substring(0, 300)}...`);
    // Continue with the conversation or process the scraped content as needed
}

import { extract } from 'scrapegraph-js';
import OpenAI from 'openai';
import { z } from 'zod';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

const CompanyInfoSchema = z.object({
    name: z.string(),
    industry: z.string().optional(),
    description: z.string().optional()
});

const result = await extract(process.env.SGAI_APIKEY!, {
    url: 'https://stripe.com',
    prompt: 'Extract all content from this page',
});
const data = result.data?.json;

const completion = await openai.chat.completions.create({
    model: 'gpt-4o-mini',
    response_format: { type: 'json_object' },
    messages: [
        {
            role: 'system',
            content: `Extract company information from the website content. Respond ONLY with valid JSON in this exact format:
{ "name": "Company Name", "industry": "Industry", "description": "One sentence description" }`
        },
        { role: 'user', content: JSON.stringify(data) }
    ]
});

const companyInfo = CompanyInfoSchema.parse(
    JSON.parse(completion.choices[0].message.content!)
);

console.log(companyInfo);