Building an AI-Powered Crawler

import * as cheerio from 'cheerio';

async function scrape(url: string) {
  const res = await fetch(url);
  const html = await res.text();
  const $ = cheerio.load(html);

  // Extract text
  const title = $('h1').text();
  const content = $('article').text();
  const links = $('a').map((_, el) => $(el).attr('href')).get();

  return { title, content, links };
}

import { chromium } from 'playwright';

async function scrapeWithBrowser(url: string) {
  const browser = await chromium.launch();
  const page = await browser.newPage();
  await page.goto(url, { waitUntil: 'networkidle' });

  const content = await page.textContent('main');
  await browser.close();

  return content;
}

async function crawlSitemap(sitemapUrl: string) {
  const res = await fetch(sitemapUrl);
  const xml = await res.text();

  // Extract URLs from sitemap.xml
  const urls = xml.match(/<loc>(.*?)<\/loc>/g)
    ?.map(m => m.replace(/<\/?loc>/g, '')) || [];

  // Crawl each URL (with rate limiting)
  const results = [];
  for (const url of urls) {
    results.push(await scrape(url));
    await sleep(1000); // 1 second interval
  }

  return results;
}

const prompt = `The following is content crawled from a web page.

Please extract the key information:
1. Topic/category
2. Key points (3-5)
3. Important numbers/data
4. Related keywords

Web page content:
${crawledContent}`;

const prompt = `The following is information about the same product collected from multiple websites.

Please create a comparison table:
- Price
- Key specs
- Pros and cons
- Recommended for

Collected data:
${JSON.stringify(products, null, 2)}`;

const prompt = `The following is a list of tech blog posts collected over the past week.

Please analyze the trends:
1. Top 10 most-mentioned technologies/keywords
2. Newly emerging trends
3. Fading trends
4. Weekly summary (3-5 lines)

Collected data:
${articles.map(a => `[${a.date}] ${a.title}: ${a.summary}`).join('\n')}`;

async function monitorCompetitors() {
  const competitors = [
    { name: 'CompanyA', url: 'https://companya.com/blog' },
    { name: 'CompanyB', url: 'https://companyb.com/changelog' },
  ];

  for (const comp of competitors) {
    const current = await scrape(comp.url);
    const previous = loadPrevious(comp.name); // Yesterday's crawl result

    if (current !== previous) {
      const analysis = await analyzeChanges(previous, current);
      await sendSlackNotification(comp.name, analysis);
      saveCurrent(comp.name, current);
    }
  }
}