MEGA-llms.txt!!!!!

import Firecrawl from "@mendable/firecrawl-js";
import * as cheerio from "cheerio";
import { parseArgs } from "node:util";

const { values } = parseArgs({
  options: {
    "max-pages": { type: "string", default: "0" },
  },
  strict: true,
});

const maxPages = Number(values["max-pages"]);
if (!Number.isFinite(maxPages) || !Number.isInteger(maxPages) || maxPages < 0) {
  throw new Error(
    "OUT_OF_SCOPE: --max-pages must be a non-negative integer (0 = all pages)"
  );
}

const BASE_URL = "https://directory.llmstxt.cloud/directory";

const firecrawl = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY });

const log = (msg: string) => process.stderr.write(msg + "\n");

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

interface Entry {
  name: string;
  url: string | null;
  llmsTxtUrl: string | null;
  llmsTxtTokens: number | null;
  llmsFullTxtUrl: string | null;
  llmsFullTxtTokens: number | null;
}

/** "892 tokens" -> 892, "10K tokens" -> 10000, "13M tokens" -> 13000000 */
function parseTokens(text: string): number | null {
  const m = text.trim().match(/^([\d.,]+)\s*([KMB])?\s*tokens?$/i);
  if (!m) return null;
  const n = parseFloat(m[1].replace(/,/g, ""));
  if (!Number.isFinite(n)) return null;
  const mult =
    m[2] === undefined
      ? 1
      : { K: 1e3, M: 1e6, B: 1e9 }[m[2].toUpperCase() as "K" | "M" | "B"];
  return Math.round(n * mult);
}

function pageUrl(page: number): string {
  return page === 1 ? BASE_URL : `${BASE_URL}?page=${page}`;
}

/** Parse one directory page's HTML into entries (one <article> card per site). */
function parseEntries(html: string, pageNum: number): Entry[] {
  const $ = cheerio.load(html);
  const entries: Entry[] = [];

  $("article").each((_, el) => {
    const card = $(el);
    const name = card.find("h3").first().text().trim();
    if (!name) return;

    const siteAnchor = card
      .find('a[target="_blank"]')
      .filter((_, a) => {
        const href = $(a).attr("href") ?? "";
        return !/llms(-full)?\.txt/i.test(href);
      })
      .first();
    const url = siteAnchor.attr("href")?.trim() ?? null;

    let llmsTxtUrl: string | null = null;
    let llmsTxtTokens: number | null = null;
    let llmsFullTxtUrl: string | null = null;
    let llmsFullTxtTokens: number | null = null;

    card.find("div").each((_, cellEl) => {
      const cell = $(cellEl);
      const label = cell.children("span").first().text().trim().toLowerCase();
      if (label !== "/llms.txt" && label !== "/full") return;
      const link = cell.children("a").first();
      if (!link.length) return; // "N/A" cell
      const href = link.attr("href")?.trim() ?? null;
      const tokens = parseTokens(link.text());
      if (label === "/llms.txt") {
        llmsTxtUrl = href;
        llmsTxtTokens = tokens;
      } else {
        llmsFullTxtUrl = href;
        llmsFullTxtTokens = tokens;
      }
    });

    entries.push({
      name,
      url,
      llmsTxtUrl,
      llmsTxtTokens,
      llmsFullTxtUrl,
      llmsFullTxtTokens,
    });
  });

  if (entries.length === 0) {
    throw new Error(`no directory rows found on page ${pageNum}`);
  }
  return entries;
}

/** Total page count from the numeric pagination links (?page=N), with the
 *  "Showing X-Y of Z" footer as a fallback. */
function parseTotalPages(html: string, entriesOnPage: number): number {
  const $ = cheerio.load(html);
  let max = 0;
  $('a[href*="page="]').each((_, a) => {
    const m = ($(a).attr("href") ?? "").match(/[?&]page=(\d+)/);
    if (m) max = Math.max(max, parseInt(m[1], 10));
  });
  if (max > 0) return max;

  const showing = $.root()
    .text()
    .match(/Showing\s+[\d,]+\s*-\s*[\d,]+\s+of\s+([\d,]+)/i);
  if (showing && entriesOnPage > 0) {
    const total = parseInt(showing[1].replace(/,/g, ""), 10);
    if (Number.isFinite(total) && total > 0) {
      return Math.ceil(total / entriesOnPage);
    }
  }

  throw new Error(
    "could not determine total page count from pagination links on page 1"
  );
}

async function fetchHtml(page: number): Promise<string> {
  const url = pageUrl(page);
  let lastError = "";
  for (let attempt = 1; attempt <= 4; attempt++) {
    try {
      const doc = await firecrawl.scrape(url, {
        formats: ["html"],
        maxAge: 0,
        integration: "prometheus",
      });
      if (doc.html && doc.html.length > 0) return doc.html;
      lastError = "empty response";
    } catch (e) {
      lastError = e instanceof Error ? e.message : String(e);
      log(`scrape attempt ${attempt} for page ${page} failed: ${lastError}`);
    }
    if (attempt < 4) await sleep(attempt * 15000);
  }
  throw new Error(`no HTML returned for directory page ${page} (${lastError})`);
}

async function main() {
  log(`fetching page 1: ${pageUrl(1)}`);
  const firstHtml = await fetchHtml(1);
  const firstEntries = parseEntries(firstHtml, 1);
  const totalPages = parseTotalPages(firstHtml, firstEntries.length);
  log(`page 1: ${firstEntries.length} entries, ${totalPages} total pages`);

  const pagesToScrape =
    maxPages === 0 ? totalPages : Math.min(maxPages, totalPages);

  const htmlByPage = new Map<number, string>([[1, firstHtml]]);

  if (pagesToScrape > 1) {
    const remaining: number[] = [];
    for (let p = 2; p <= pagesToScrape; p++) remaining.push(p);
    const urls = remaining.map(pageUrl);
    log(`batch scraping ${urls.length} remaining pages`);
    const job = await firecrawl.batchScrape(urls, {
      options: { formats: ["html"], maxAge: 0 },
      integration: "prometheus",
      pollInterval: 5,
      timeout: 540,
    });
    log(`batch job ${job.id}: ${job.status} (${job.completed}/${job.total})`);
    for (const doc of job.data ?? []) {
      const src = doc.metadata?.sourceURL ?? doc.metadata?.url ?? "";
      const m = String(src).match(/[?&]page=(\d+)/);
      if (m && doc.html) htmlByPage.set(parseInt(m[1], 10), doc.html);
    }
    // retry any pages the batch job missed, one by one
    for (const p of remaining) {
      if (!htmlByPage.has(p)) {
        log(`page ${p} missing from batch result, retrying individually`);
        htmlByPage.set(p, await fetchHtml(p));
      }
    }
  }

  const entries: Entry[] = [];
  for (let p = 1; p <= pagesToScrape; p++) {
    const html = htmlByPage.get(p);
    if (!html) throw new Error(`no HTML returned for directory page ${p}`);
    const pageEntries = p === 1 ? firstEntries : parseEntries(html, p);
    log(`page ${p}: ${pageEntries.length} entries`);
    entries.push(...pageEntries);
  }

  const out = {
    source: BASE_URL,
    totalPages,
    pagesScraped: pagesToScrape,
    totalEntries: entries.length,
    entries,
  };
  process.stdout.write(JSON.stringify(out));
}

main().catch((e) => {
  log(e instanceof Error ? e.stack ?? e.message : String(e));
  process.exit(1);
});