Bankrate Best Credit Cards

import Firecrawl from "@mendable/firecrawl-js";
import * as cheerio from "cheerio";

const apiKey = process.env.FIRECRAWL_API_KEY;
if (!apiKey) {
  console.error("FIRECRAWL_API_KEY is not set");
  process.exit(1);
}
const firecrawl = new Firecrawl({ apiKey });

async function main() {
  const doc = await firecrawl.scrape(
    "https://www.bankrate.com/credit-cards/best-credit-cards/",
    { formats: ["html"], integration: "prometheus" }
  );
  const html = doc.html ?? "";
  if (!html) {
    throw new Error("no html returned from Bankrate best-credit-cards page");
  }
  const $ = cheerio.load(html);

  const cardEls = $('div[id]')
    .toArray()
    .filter((el) => /^card\d+$/.test($(el).attr("id") || ""));

  if (cardEls.length === 0) {
    throw new Error("no editor card containers found on the page");
  }

  const cards = cardEls.map((el) => {
    const $c = $(el);
    const name = $c.find("h2").first().text().trim().replace(/\s+/g, " ");

    const apply =
      $c.find('a:contains("Apply now")').first().attr("href") || "";

    const score = $c
      .find("span")
      .filter(function () {
        return /^\d(\.\d)?$/.test($(this).text().trim());
      })
      .first()
      .text()
      .trim();

    // ordered <p> texts for label->value pairing
    const ps: string[] = [];
    $c.find("p").each((_, p) => {
      const t = $(p).text().trim().replace(/\s+/g, " ");
      if (t) ps.push(t);
    });
    const valAfter = (label: string) => {
      const i = ps.findIndex((p) => p === label);
      return i >= 0 && i + 1 < ps.length ? ps[i + 1] : "";
    };

    const introOffer = valAfter("Intro offer");
    let rewardsRate = valAfter("Rewards rate");
    const annualFee = valAfter("Annual fee");
    const regularApr = valAfter("Regular APR");
    const bestFor = ps.find((p) => /^Best/.test(p)) || "";

    // issuer from "on <issuer>'s secure site"
    const secureP = $c
      .find("p")
      .filter(function () {
        return /secure site/.test($(this).text());
      })
      .first()
      .text();
    const issuerMatch = secureP.match(/on (.*?)'s secure site/);
    const issuer = issuerMatch ? issuerMatch[1].trim() : "";

    // rewards summary: use the concise rate; otherwise build from Reward Details list items
    let rewardsSummary = rewardsRate;
    if (!rewardsSummary) {
      const rdItems = $c
        .find('#reward-details li, [data-testid*="reward-rate"]')
        .closest("li")
        .toArray();
      const parts: string[] = [];
      const seen = new Set<string>();
      $c.find('[data-testid*="reward-rate"]').each((_, li) => {
        const $li = $(li).closest("li");
        const rate = $li.find("span").first().text().trim();
        const desc = $(li).text().trim().replace(/\s+/g, " ");
        const key = rate + "|" + desc;
        if (!seen.has(key)) {
          seen.add(key);
          parts.push(`${rate} ${desc}`);
        }
      });
      if (parts.length) rewardsSummary = parts.join("; ");
    }
    if (!rewardsSummary) rewardsSummary = "N/A";

    const intro = introOffer || "N/A";
    const fee = annualFee || "N/A";
    const apr = regularApr || "N/A";

    return {
      rank: Number($c.attr("id")!.replace("card", "")),
      name,
      issuer,
      bestFor,
      bankrateScore: score || null,
      introOffer: intro,
      rewardsSummary,
      annualFee: fee,
      regularApr: apr,
      applyUrl: apply,
    };
  });

  cards.sort((a, b) => a.rank - b.rank);
  for (const c of cards) delete (c as Partial<typeof c>).rank;

  const out = {
    source: "Bankrate — Best Credit Cards (editor's top picks)",
    sourceUrl: "https://www.bankrate.com/credit-cards/best-credit-cards/",
    retrievedAt: new Date().toISOString(),
    count: cards.length,
    note:
      "Bankrate's editor top-picks list contains 15 cards; requested 25 but only 15 are published on this page.",
    cards,
  };

  process.stdout.write(JSON.stringify(out));
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});