Replace full-site crawls with sitemap extraction
Instead of crawling an entire website page by page, use sitemap extraction to get all target URLs in a single API call. This is faster, cheaper, and more polite to the target server.
import requests
# Get all product URLs from an e-commerce site
resp = requests.post(
"https://sitemapkit.com/api/v1/sitemap/full",
headers={"x-api-key": "YOUR_API_KEY"},
json={"url": "shop.example.com"}
)
urls = resp.json()["urls"]
product_urls = [u["loc"] for u in urls if "/products/" in u["loc"]]
print(f"Found {len(product_urls)} product pages to scrape")Free tier includes 100 API calls/month. No credit card required.