from pydantic import BaseModel, Field
from typing import List, Optional
from datetime import datetime
from scrapegraph_py import Client

# Schema for news article data
class NewsArticle(BaseModel):
    title: str = Field(description="Article title")
    content: str = Field(description="Article content")
    summary: Optional[str] = Field(description="Article summary")
    author: Optional[str] = Field(description="Article author")
    publication_date: str = Field(description="Publication date")
    source_url: str = Field(description="Source URL")
    category: Optional[str] = Field(description="Article category")
    tags: Optional[List[str]] = Field(description="Article tags")
    image_url: Optional[str] = Field(description="Featured image URL")

# Schema for news aggregation results
class NewsAggregationResult(BaseModel):
    articles: List[NewsArticle] = Field(description="List of aggregated articles")
    total_articles: int = Field(description="Total number of articles collected")
    sources: List[str] = Field(description="List of source URLs")
    timestamp: str = Field(description="Aggregation timestamp")

client = Client()

# Define news sources to aggregate
news_sources = [
    "https://news-site1.com",
    "https://news-site2.com",
    "https://news-site3.com"
]

# Aggregate news from multiple sources
aggregated_results = []

for source in news_sources:
    response = client.smartscraper(
        website_url=source,
        user_prompt="Extract all news articles from the homepage, including title, content, author, publication date, category, and tags. Also extract featured images if available.",
        output_schema=NewsAggregationResult
    )
    aggregated_results.append(response)

# Process and display results
total_articles = sum(result.total_articles for result in aggregated_results)
print(f"Aggregated {total_articles} articles from {len(news_sources)} sources\n")

for result in aggregated_results:
    print(f"Source: {result.sources[0]}")
    print(f"Articles: {result.total_articles}")
    
    for article in result.articles:
        print(f"\nTitle: {article.title}")
        print(f"Author: {article.author or 'Unknown'}")
        print(f"Date: {article.publication_date}")
        if article.category:
            print(f"Category: {article.category}")
        if article.tags:
            print(f"Tags: {', '.join(article.tags)}")

from pydantic import BaseModel, Field
from typing import List, Optional
from datetime import datetime, timedelta
from scrapegraph_py import Client
import time  # For job polling

# Schema for blog post data
class BlogPost(BaseModel):
    title: str = Field(description="Blog post title")
    content: str = Field(description="Blog post content")
    excerpt: Optional[str] = Field(description="Post excerpt or summary")
    author: str = Field(description="Post author")
    publication_date: str = Field(description="Publication date")
    url: str = Field(description="Post URL")
    categories: Optional[List[str]] = Field(description="Post categories")
    tags: Optional[List[str]] = Field(description="Post tags")
    comments_count: Optional[int] = Field(description="Number of comments")
    reading_time: Optional[int] = Field(description="Estimated reading time in minutes")

# Schema for blog monitoring results
class BlogMonitorResult(BaseModel):
    posts: List[BlogPost] = Field(description="List of blog posts")
    total_posts: int = Field(description="Total number of posts found")
    blog_url: str = Field(description="Blog homepage URL")
    last_updated: str = Field(description="Last monitoring timestamp")

client = Client()

# Start the crawler job
job = client.smartcrawler_initiate(
    url="https://example-blog.com",
    user_prompt="Extract all blog posts from the last 7 days, including title, content, author, publication date, categories, and metadata. Calculate estimated reading time based on content length.",
    extraction_mode="ai",
    depth=2,  # Crawl up to 2 levels deep
    same_domain_only=True
)

# Wait for job completion and get results
job_id = job.job_id
while True:
    status = client.smartcrawler_get_status(job_id)
    if status.state == "completed":
        response = status.result
        break
    elif status.state in ["failed", "cancelled"]:
        print(f"Job failed: {status.error}")
        break
    time.sleep(5)  # Wait 5 seconds before checking again

# Process the crawled content if successful
if response:
    print(f"Blog: {response.blog_url}")
    print(f"Total Posts Found: {response.total_posts}")
    print(f"Last Updated: {response.last_updated}\n")

    for post in response.posts:
        # Check if post is recent
        post_date = datetime.strptime(post.publication_date, "%Y-%m-%d")
        if post_date > datetime.now() - timedelta(days=7):
            print(f"Title: {post.title}")
            print(f"Author: {post.author}")
            print(f"Published: {post.publication_date}")
            print(f"Reading Time: {post.reading_time} minutes")
            if post.categories:
                print(f"Categories: {', '.join(post.categories)}")
            if post.tags:
                print(f"Tags: {', '.join(post.tags)}")
            if post.excerpt:
                print(f"\nExcerpt: {post.excerpt}")
            print(f"URL: {post.url}\n")