SitemapKit + Crawl4AI

import requests
from crawl4ai import AsyncWebCrawler

# Step 1: Get all URLs via SitemapKit
resp = requests.post(
    "https://sitemapkit.com/api/v1/sitemap/full",
    headers={"x-api-key": "YOUR_API_KEY", "Content-Type": "application/json"},
    json={"url": "docs.example.com"}
)
urls = [u["loc"] for u in resp.json()["urls"]]
print(f"Found {len(urls)} URLs to crawl")

# Step 2: Crawl with Crawl4AI for LLM-ready content
async with AsyncWebCrawler() as crawler:
    for url in urls[:50]:
        result = await crawler.arun(url=url)
        if result.success:
            # result.markdown contains clean, LLM-ready text
            save_to_dataset(url, result.markdown)