SEC Filings And Earnings Transcripts

import Firecrawl from "@mendable/firecrawl-js";
import * as cheerio from "cheerio";
import { parseArgs } from "node:util";

const apiKey = process.env.FIRECRAWL_API_KEY;
if (!apiKey) {
  console.error("FIRECRAWL_API_KEY is not set");
  process.exit(1);
}

const firecrawl = new Firecrawl({ apiKey });

const { values: flags } = parseArgs({
  strict: true,
  options: {
    query: { type: "string" },
    "source-types": { type: "string" },
    "form-types": { type: "string" },
    "max-documents": { type: "string" },
    "output-mode": { type: "string" },
    "date-range": { type: "string" },
    "include-tables": { type: "string" },
    "include-risk-factors": { type: "string" },
    "include-mda": { type: "string" },
    "include-financial-statements": { type: "string" },
    language: { type: "string" },
    "snapshot-label": { type: "string" },
    "sort-hint": { type: "string" },
  },
});

function requiredFlag(name: string): string {
  const value = flags[name] as string | undefined;
  if (!value || !value.trim()) {
    console.error(`--${name} is required`);
    process.exit(1);
  }
  return value.trim();
}

const query = requiredFlag("query");
const sourceTypes = parseList(requiredFlag("source-types"));
const formTypes = parseList(requiredFlag("form-types"));
const maxDocuments = Number(requiredFlag("max-documents"));
const outputMode = requiredFlag("output-mode");
const dateRange = String(flags["date-range"] ?? "last 12 months");
const includeTables = parseBoolean(String(flags["include-tables"] ?? "true"), "include-tables");
const includeRiskFactors = parseBoolean(String(flags["include-risk-factors"] ?? "true"), "include-risk-factors");
const includeMda = parseBoolean(String(flags["include-mda"] ?? "true"), "include-mda");
const includeFinancialStatements = parseBoolean(String(flags["include-financial-statements"] ?? "true"), "include-financial-statements");
const language = String(flags.language ?? "English");
const snapshotLabel = String(flags["snapshot-label"] ?? new Date().toISOString());
const sortHint = String(flags["sort-hint"] ?? "newest");

if (!Number.isInteger(maxDocuments) || maxDocuments < 1 || maxDocuments > 20) {
  console.error("OUT_OF_SCOPE: --max-documents must be an integer from 1 to 20");
  process.exit(1);
}
validateList("source-types", sourceTypes, ["sec_filings", "earnings_transcripts"]);
if (formTypes.length === 0) {
  console.error("OUT_OF_SCOPE: --form-types must include at least one SEC form");
  process.exit(1);
}
if (outputMode !== "document_rows" && outputMode !== "grouped_by_company") {
  console.error("OUT_OF_SCOPE: --output-mode must be document_rows or grouped_by_company");
  process.exit(1);
}
if (language.toLowerCase() !== "english") {
  console.error("OUT_OF_SCOPE: this collector currently supports English output only");
  process.exit(1);
}
if (sortHint !== "newest") {
  console.error("OUT_OF_SCOPE: this collector currently supports --sort-hint=newest only");
  process.exit(1);
}

type SecCandidate = {
  companyName: string | null;
  ticker: string | null;
  cik: string | null;
  formType: string | null;
  filingDate: string | null;
  periodOfReport: string | null;
  accessionNumber: string | null;
  documentTitle: string | null;
  documentUrl: string | null;
  sourcePageUrl: string | null;
  secondarySourceUrls: string[];
};

type OutputRow = {
  tracking_key: string;
  snapshot_label: string;
  collected_at: string;
  company_name: string | null;
  ticker: string | null;
  cik: string | null;
  document_type: string;
  form_type: string | null;
  filing_date: string | null;
  period_of_report: string | null;
  accession_number: string | null;
  document_title: string | null;
  document_url: string | null;
  source_page_url: string | null;
  secondary_source_urls: string[];
  risk_factors_text: string | null;
  mda_text: string | null;
  business_overview_text: string | null;
  financial_statement_tables: TableOut[] | null;
  other_tables: TableOut[] | null;
  transcript_date: string | null;
  transcript_title: string | null;
  transcript_text: string | null;
  speaker_turns: SpeakerTurn[] | null;
  confidence: number;
  notes: string[];
};

type TableOut = {
  caption: string | null;
  headers: string[];
  rows: string[][];
  row_objects: Record<string, string>[];
};

type SpeakerTurn = {
  speaker: string | null;
  text: string;
};

function parseList(value: string): string[] {
  return value.split(",").map((part) => part.trim()).filter(Boolean);
}

function validateList(name: string, values: string[], allowed: string[]): void {
  const allowedSet = new Set(allowed);
  for (const value of values) {
    if (!allowedSet.has(value)) {
      console.error(`OUT_OF_SCOPE: --${name} contains unsupported value ${value}`);
      process.exit(1);
    }
  }
}

function parseBoolean(value: string, name: string): boolean {
  if (value === "true") return true;
  if (value === "false") return false;
  console.error(`OUT_OF_SCOPE: --${name} must be true or false`);
  process.exit(1);
}

function cleanText(value: string | null | undefined): string {
  return String(value ?? "").replace(/\s+/g, " ").trim();
}

function compactCik(value: string | null): string | null {
  if (!value) return null;
  const digits = value.split("").filter((char) => char >= "0" && char <= "9").join("");
  if (!digits) return null;
  return digits.padStart(10, "0");
}

function noLeadCik(value: string | null): string | null {
  const cik = compactCik(value);
  if (!cik) return null;
  const stripped = cik.replace(/^0+/, "");
  return stripped || "0";
}

function accessionNoDashes(value: string | null): string | null {
  if (!value) return null;
  return value.split("-").join("");
}

function parseJsonFromDoc(doc: any, label: string): any {
  const raw = String(doc.rawHtml ?? doc.markdown ?? "").trim();
  if (!raw) throw new Error(`no JSON returned from ${label}`);
  const fenced = raw.startsWith("```json") ? raw.slice(7, raw.lastIndexOf("```")).trim() : raw;
  return JSON.parse(fenced);
}

function dateStartForRange(range: string): string | null {
  const lower = range.toLowerCase().trim();
  const now = new Date();
  let months: number | null = null;
  if (lower === "last 12 months") months = 12;
  if (lower === "last 6 months") months = 6;
  if (lower === "last 24 months") months = 24;
  if (months === null) return null;
  const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - months, now.getUTCDate()));
  return start.toISOString().slice(0, 10);
}

async function resolveCompanyFromSec(): Promise<{ cik: string | null; ticker: string | null; name: string | null }> {
  const cik = compactCik(query);
  if (cik && query.split("").filter((char) => char >= "0" && char <= "9").length >= 4) {
    return { cik, ticker: null, name: null };
  }

  const doc = await firecrawl.scrape("https://www.sec.gov/files/company_tickers.json", {
    formats: ["rawHtml"],
    integration: "prometheus",
  });
  const data = parseJsonFromDoc(doc, "SEC company tickers");
  const entries = Object.keys(data).map((key) => data[key]);
  const q = query.toLowerCase();
  let best: any = null;
  for (let i = 0; i < entries.length; i += 1) {
    const item = entries[i];
    const ticker = String(item.ticker ?? "").toLowerCase();
    const title = String(item.title ?? "").toLowerCase();
    if (ticker === q || title === q) {
      best = item;
      break;
    }
    if (!best && (title.includes(q) || q.includes(ticker))) {
      best = item;
    }
  }
  if (!best) return { cik: null, ticker: null, name: null };
  return {
    cik: compactCik(String(best.cik_str ?? "")),
    ticker: String(best.ticker ?? "") || null,
    name: String(best.title ?? "") || null,
  };
}

async function discoverSecCandidates(): Promise<SecCandidate[]> {
  const resolved = await resolveCompanyFromSec();
  if (resolved.cik) {
    const submissionsUrl = `https://data.sec.gov/submissions/CIK${resolved.cik}.json`;
    const doc = await firecrawl.scrape(submissionsUrl, {
      formats: ["rawHtml"],
      integration: "prometheus",
    });
    const data = parseJsonFromDoc(doc, "SEC company submissions");
    return candidatesFromSubmissions(data, resolved);
  }
  return candidatesFromSearch();
}

function candidatesFromSubmissions(data: any, resolved: { cik: string | null; ticker: string | null; name: string | null }): SecCandidate[] {
  const recent = data.filings?.recent;
  if (!recent) throw new Error("SEC submissions response did not include recent filings");
  const out: SecCandidate[] = [];
  const forms = recent.form ?? [];
  const startDate = dateStartForRange(dateRange);
  const ticker = resolved.ticker ?? (Array.isArray(data.tickers) ? data.tickers[0] ?? null : null);
  const companyName = resolved.name ?? data.name ?? null;
  const cik = compactCik(String(data.cik ?? resolved.cik ?? ""));
  const cikNoLead = noLeadCik(cik);

  for (let i = 0; i < forms.length && out.length < maxDocuments; i += 1) {
    const form = String(forms[i] ?? "");
    if (!formTypes.includes(form)) continue;
    const filingDate = String(recent.filingDate?.[i] ?? "");
    if (startDate && filingDate && filingDate < startDate) continue;
    const accession = String(recent.accessionNumber?.[i] ?? "");
    const primaryDocument = String(recent.primaryDocument?.[i] ?? "");
    const accessionCompact = accessionNoDashes(accession);
    const base = cikNoLead && accessionCompact ? `https://www.sec.gov/Archives/edgar/data/${cikNoLead}/${accessionCompact}` : null;
    out.push({
      companyName,
      ticker,
      cik,
      formType: form,
      filingDate: filingDate || null,
      periodOfReport: String(recent.reportDate?.[i] ?? "") || null,
      accessionNumber: accession || null,
      documentTitle: String(recent.primaryDocDescription?.[i] ?? form) || form,
      documentUrl: base && primaryDocument ? `${base}/${primaryDocument}` : null,
      sourcePageUrl: base && accession ? `${base}/${accession}-index.html` : submissionsUrlFor(cik),
      secondarySourceUrls: [submissionsUrlFor(cik)].filter(Boolean) as string[],
    });
  }
  return out;
}

async function candidatesFromSearch(): Promise<SecCandidate[]> {
  const formQuery = formTypes.map((form) => encodeURIComponent(form)).join("%2C");
  const searchUrl = `https://efts.sec.gov/LATEST/search-index?keys=${encodeURIComponent(query)}&category=form-cat1&forms=${formQuery}&from=0&size=${Math.min(maxDocuments, 10)}`;
  const doc = await firecrawl.scrape(searchUrl, {
    formats: ["rawHtml"],
    integration: "prometheus",
  });
  const data = parseJsonFromDoc(doc, "SEC EDGAR full-text search");
  const hits = data.hits?.hits ?? [];
  const out: SecCandidate[] = [];
  const startDate = dateStartForRange(dateRange);
  for (let i = 0; i < hits.length && out.length < maxDocuments; i += 1) {
    const src = hits[i]?._source ?? {};
    const fileDate = String(src.file_date ?? "");
    if (startDate && fileDate && fileDate < startDate) continue;
    const form = String(src.form ?? src.root_forms?.[0] ?? "");
    if (!formTypes.includes(form) && !formTypes.includes(String(src.root_forms?.[0] ?? ""))) continue;
    const accession = String(src.adsh ?? "");
    const cik = compactCik(String(src.ciks?.[0] ?? ""));
    const cikNoLead = noLeadCik(cik);
    const accessionCompact = accessionNoDashes(accession);
    const fileName = String(hits[i]?._id ?? "").split(":")[1] ?? "";
    const base = cikNoLead && accessionCompact ? `https://www.sec.gov/Archives/edgar/data/${cikNoLead}/${accessionCompact}` : null;
    const display = String(src.display_names?.[0] ?? "");
    out.push({
      companyName: display.split("  (")[0] || null,
      ticker: tickerFromDisplay(display),
      cik,
      formType: form || null,
      filingDate: fileDate || null,
      periodOfReport: String(src.period_ending ?? "") || null,
      accessionNumber: accession || null,
      documentTitle: String(src.file_description ?? form) || null,
      documentUrl: base && fileName ? `${base}/${fileName}` : null,
      sourcePageUrl: base && accession ? `${base}/${accession}-index.html` : searchUrl,
      secondarySourceUrls: [searchUrl],
    });
  }
  return out;
}

function submissionsUrlFor(cik: string | null): string | null {
  return cik ? `https://data.sec.gov/submissions/CIK${compactCik(cik)}.json` : null;
}

function tickerFromDisplay(display: string): string | null {
  const marker = "  (";
  const start = display.indexOf(marker);
  if (start < 0) return null;
  const end = display.indexOf(")", start);
  if (end < 0) return null;
  const inside = display.slice(start + marker.length, end);
  if (inside.includes("CIK")) return null;
  return inside.split(",")[0].trim() || null;
}

async function extractSecRow(candidate: SecCandidate, collectedAt: string): Promise<OutputRow> {
  const notes: string[] = [];
  let risk: string | null = null;
  let mda: string | null = null;
  let business: string | null = null;
  let financialTables: TableOut[] | null = null;
  let otherTables: TableOut[] | null = null;

  if (!candidate.documentUrl) {
    notes.push("SEC filing document URL was unavailable from discovery metadata.");
  } else {
    const doc = await firecrawl.scrape(candidate.documentUrl, {
      formats: ["rawHtml"],
      integration: "prometheus",
    });
    const rawHtml = String((doc as any).rawHtml ?? "");
    if (!rawHtml) throw new Error("SEC filing document scrape returned no raw HTML");
    const $ = cheerio.load(rawHtml);
    $("script, style, noscript").remove();
    const bodyText = cleanText($("body").text());
    if (includeRiskFactors) risk = sectionBetween(bodyText, ["item 1a. risk factors", "item 1a risk factors"], ["item 1b.", "item 1b ", "item 2."]);
    if (includeMda) mda = sectionBetween(bodyText, ["item 7. management", "item 7 management", "item 2. management", "item 2 management"], ["item 7a.", "item 3.", "item 8."]);
    business = sectionBetween(bodyText, ["item 1. business", "item 1 business"], ["item 1a.", "item 2."]);
    if (!risk && includeRiskFactors) notes.push("Risk factors section could not be isolated reliably from visible filing text.");
    if (!mda && includeMda) notes.push("MD&A section could not be isolated reliably from visible filing text.");
    if (!business) notes.push("Business overview section could not be isolated reliably from visible filing text.");
    if (includeTables) {
      const tables = extractTables($, includeFinancialStatements);
      financialTables = tables.financial;
      otherTables = tables.other;
      if (financialTables.length === 0 && includeFinancialStatements) notes.push("No financial statement tables were confidently classified from visible tables.");
    }
  }

  return emptyRow({
    trackingKey: trackingKey(candidate.cik, candidate.accessionNumber ?? candidate.documentUrl ?? "sec"),
    collectedAt,
    documentType: "sec_filing",
    companyName: candidate.companyName,
    ticker: candidate.ticker,
    cik: candidate.cik,
    formType: candidate.formType,
    filingDate: candidate.filingDate,
    periodOfReport: candidate.periodOfReport,
    accessionNumber: candidate.accessionNumber,
    documentTitle: candidate.documentTitle,
    documentUrl: candidate.documentUrl,
    sourcePageUrl: candidate.sourcePageUrl,
    secondarySourceUrls: candidate.secondarySourceUrls,
    risk,
    mda,
    business,
    financialTables,
    otherTables,
    confidence: candidate.documentUrl ? 0.86 : 0.6,
    notes,
  });
}

function sectionBetween(text: string, starts: string[], ends: string[]): string | null {
  const lower = text.toLowerCase();
  let bestStart = -1;
  let startLength = 0;
  for (let i = 0; i < starts.length; i += 1) {
    const found = lower.indexOf(starts[i]);
    if (found >= 0 && (bestStart < 0 || found < bestStart)) {
      bestStart = found;
      startLength = starts[i].length;
    }
  }
  if (bestStart < 0) return null;
  let bestEnd = text.length;
  const from = bestStart + startLength;
  for (let i = 0; i < ends.length; i += 1) {
    const found = lower.indexOf(ends[i], from);
    if (found > from && found < bestEnd) bestEnd = found;
  }
  const section = text.slice(bestStart, bestEnd).trim();
  if (section.length < 200) return null;
  return limitText(section, 14000);
}

function extractTables($: cheerio.CheerioAPI, includeFinancial: boolean): { financial: TableOut[]; other: TableOut[] } {
  const financial: TableOut[] = [];
  const other: TableOut[] = [];
  const maxTables = 18;
  const tableNodes = $("table").toArray();
  for (let i = 0; i < tableNodes.length && i < maxTables; i += 1) {
    const table = $(tableNodes[i]);
    const rows: string[][] = [];
    const trNodes = table.find("tr").toArray();
    for (let r = 0; r < trNodes.length && r < 40; r += 1) {
      const cells = $(trNodes[r]).find("th,td").toArray().map((cell) => cleanText($(cell).text())).filter(Boolean);
      if (cells.length > 0) rows.push(cells.slice(0, 12));
    }
    if (rows.length < 2) continue;
    const caption = cleanText(table.find("caption").first().text()) || null;
    const headers = rows[0];
    const rowObjects = rows.slice(1, 20).map((row) => rowToObject(headers, row));
    const out = { caption, headers, rows: rows.slice(1, 20), row_objects: rowObjects };
    const tableText = cleanText(`${caption ?? ""} ${rows.slice(0, 5).map((row) => row.join(" ")).join(" ")}`).toLowerCase();
    const isFinancial = includesAny(tableText, ["statement of operations", "balance sheet", "cash flow", "income statement", "assets", "liabilities", "revenue", "net income"]);
    if (includeFinancial && isFinancial && financial.length < 8) financial.push(out);
    if ((!isFinancial || !includeFinancial) && other.length < 6) other.push(out);
  }
  return { financial, other };
}

function rowToObject(headers: string[], row: string[]): Record<string, string> {
  const out: Record<string, string> = {};
  for (let i = 0; i < headers.length && i < row.length; i += 1) {
    const key = headers[i] || `column_${i + 1}`;
    out[key] = row[i] ?? "";
  }
  return out;
}

function includesAny(text: string, needles: string[]): boolean {
  for (let i = 0; i < needles.length; i += 1) {
    if (text.includes(needles[i])) return true;
  }
  return false;
}

async function discoverTranscriptRows(existingCompany: { companyName: string | null; ticker: string | null; cik: string | null }, collectedAt: string, remaining: number): Promise<OutputRow[]> {
  const rows: OutputRow[] = [];
  if (remaining <= 0) return rows;
  const transcriptQuery = `${existingCompany.companyName ?? query} ${existingCompany.ticker ?? ""} earnings call transcript`;
  const search = await firecrawl.search(transcriptQuery, {
    limit: Math.min(5, Math.max(1, remaining + 2)),
    integration: "prometheus",
  });
  const searchItems = (search as any).data ?? (search as any).web ?? [];
  const candidates = searchItems.filter((item: any) => isTranscriptCandidate(item)).sort((a: any, b: any) => transcriptScore(b) - transcriptScore(a));
  const maxTranscriptScrapes = Math.min(3, remaining, candidates.length);
  for (let i = 0; i < maxTranscriptScrapes && rows.length < remaining; i += 1) {
    const item = candidates[i];
    const url = String(item.url ?? "");
    if (!url) continue;
    const doc = await firecrawl.scrape(url, {
      formats: ["markdown"],
      integration: "prometheus",
    });
    const markdown = cleanTranscriptText(String((doc as any).markdown ?? ""));
    if (markdown.length < 600 || looksPaywalled(markdown)) continue;
    const speakerTurns = extractSpeakerTurnsBounded(markdown);
    rows.push(emptyRow({
      trackingKey: trackingKey(existingCompany.cik, url),
      collectedAt,
      documentType: "earnings_transcript",
      companyName: existingCompany.companyName,
      ticker: existingCompany.ticker,
      cik: existingCompany.cik,
      formType: null,
      filingDate: null,
      periodOfReport: null,
      accessionNumber: null,
      documentTitle: cleanText(String(item.title ?? "")) || "Earnings call transcript",
      documentUrl: url,
      sourcePageUrl: url,
      secondarySourceUrls: [],
      risk: null,
      mda: null,
      business: null,
      financialTables: null,
      otherTables: null,
      transcriptDate: dateFromText(String(item.title ?? "") + " " + markdown),
      transcriptTitle: cleanText(String(item.title ?? "")) || null,
      transcriptText: limitText(markdown, 18000),
      speakerTurns,
      confidence: speakerTurns.length > 0 ? 0.74 : 0.62,
      notes: speakerTurns.length > 0 ? ["Transcript source is publicly visible and speaker turns were parsed with bounded line scanning."] : ["Transcript source is publicly visible, but speaker turns could not be confidently separated."],
    }));
  }
  return rows;
}

function isTranscriptCandidate(item: any): boolean {
  const text = `${item.title ?? ""} ${item.description ?? ""} ${item.url ?? ""}`.toLowerCase();
  if (!text.includes("transcript")) return false;
  if (includesAny(text, ["premium", "subscribe", "paywall"])) return false;
  return includesAny(text, ["earnings", "quarter", "q1", "q2", "q3", "q4", "call"]);
}

function transcriptScore(item: any): number {
  const text = `${item.title ?? ""} ${item.description ?? ""} ${item.url ?? ""}`.toLowerCase();
  let score = 0;
  if (text.includes("call-transcripts")) score += 5;
  if (text.includes("earnings transcript")) score += 4;
  if (text.includes("/20")) score += 2;
  if (text.includes("investor")) score += 1;
  if (text.includes("earnings-calls")) score -= 3;
  if (text.includes("seekingalpha")) score -= 2;
  if (text.includes("morningstar")) score -= 2;
  return score;
}

function cleanTranscriptText(text: string): string {
  return text.split("\n").map((line) => line.trim()).filter(Boolean).join("\n");
}

function looksPaywalled(text: string): boolean {
  const lower = text.toLowerCase();
  return includesAny(lower, ["subscribe to continue", "sign in to continue", "premium article", "access this transcript"]);
}

function extractSpeakerTurnsBounded(text: string): SpeakerTurn[] {
  const lines = text.split("\n").slice(0, 500);
  const turns: SpeakerTurn[] = [];
  for (let i = 0; i < lines.length && turns.length < 80; i += 1) {
    const line = cleanText(lines[i]);
    if (!line || line.length > 160) continue;
    const colon = line.indexOf(":");
    if (colon > 1 && colon < 60) {
      const speaker = line.slice(0, colon).trim();
      const spoken = line.slice(colon + 1).trim();
      if (speaker.length > 1 && spoken.length > 20) {
        turns.push({ speaker, text: limitText(spoken, 1200) });
      }
      continue;
    }
    const next = cleanText(lines[i + 1] ?? "");
    if (isLikelySpeakerName(line) && next.length > 40) {
      turns.push({ speaker: line, text: limitText(next, 1200) });
      i += 1;
    }
  }
  return turns;
}

function isLikelySpeakerName(line: string): boolean {
  if (line.length < 3 || line.length > 70) return false;
  if (includesAny(line.toLowerCase(), ["operator", "analyst", "question-and-answer", "presentation"])) return true;
  const words = line.split(" ");
  if (words.length < 2 || words.length > 5) return false;
  let capitalized = 0;
  for (let i = 0; i < words.length; i += 1) {
    const first = words[i].charAt(0);
    if (first >= "A" && first <= "Z") capitalized += 1;
  }
  return capitalized >= Math.min(2, words.length);
}

function dateFromText(text: string): string | null {
  const months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
  for (let i = 0; i < months.length; i += 1) {
    const month = months[i];
    const idx = text.indexOf(month);
    if (idx < 0) continue;
    const snippet = text.slice(idx, idx + 40);
    const parts = snippet.replace(",", " ").split(" ").filter(Boolean);
    if (parts.length >= 3) return parts.slice(0, 3).join(" ");
  }
  return null;
}

function limitText(text: string, max: number): string {
  if (text.length <= max) return text;
  return `${text.slice(0, max).trim()} [truncated]`;
}

function trackingKey(cik: string | null, value: string): string {
  return `${snapshotLabel}:${cik ?? "unknown"}:${value}`.replace(/\s+/g, "_");
}

function emptyRow(input: {
  trackingKey: string;
  collectedAt: string;
  documentType: string;
  companyName: string | null;
  ticker: string | null;
  cik: string | null;
  formType: string | null;
  filingDate: string | null;
  periodOfReport: string | null;
  accessionNumber: string | null;
  documentTitle: string | null;
  documentUrl: string | null;
  sourcePageUrl: string | null;
  secondarySourceUrls: string[];
  risk: string | null;
  mda: string | null;
  business: string | null;
  financialTables: TableOut[] | null;
  otherTables: TableOut[] | null;
  transcriptDate?: string | null;
  transcriptTitle?: string | null;
  transcriptText?: string | null;
  speakerTurns?: SpeakerTurn[] | null;
  confidence: number;
  notes: string[];
}): OutputRow {
  return {
    tracking_key: input.trackingKey,
    snapshot_label: snapshotLabel,
    collected_at: input.collectedAt,
    company_name: input.companyName,
    ticker: input.ticker,
    cik: input.cik,
    document_type: input.documentType,
    form_type: input.formType,
    filing_date: input.filingDate,
    period_of_report: input.periodOfReport,
    accession_number: input.accessionNumber,
    document_title: input.documentTitle,
    document_url: input.documentUrl,
    source_page_url: input.sourcePageUrl,
    secondary_source_urls: input.secondarySourceUrls,
    risk_factors_text: input.risk,
    mda_text: input.mda,
    business_overview_text: input.business,
    financial_statement_tables: input.financialTables,
    other_tables: input.otherTables,
    transcript_date: input.transcriptDate ?? null,
    transcript_title: input.transcriptTitle ?? null,
    transcript_text: input.transcriptText ?? null,
    speaker_turns: input.speakerTurns ?? null,
    confidence: input.confidence,
    notes: input.notes,
  };
}

function groupedOutput(rows: OutputRow[]) {
  const companies: Record<string, { company_name: string | null; ticker: string | null; cik: string | null; documents: OutputRow[] }> = {};
  for (let i = 0; i < rows.length; i += 1) {
    const row = rows[i];
    const key = row.cik ?? row.company_name ?? "unknown";
    if (!companies[key]) {
      companies[key] = { company_name: row.company_name, ticker: row.ticker, cik: row.cik, documents: [] };
    }
    companies[key].documents.push(row);
  }
  return {
    snapshot_label: snapshotLabel,
    collected_at: rows[0]?.collected_at ?? new Date().toISOString(),
    query,
    companies: Object.keys(companies).map((key) => companies[key]),
  };
}

async function main() {
  const collectedAt = new Date().toISOString();
  const rows: OutputRow[] = [];
  let firstCompany = { companyName: null as string | null, ticker: null as string | null, cik: null as string | null };

  if (sourceTypes.includes("sec_filings") && rows.length < maxDocuments) {
    const candidates = await discoverSecCandidates();
    if (candidates.length === 0) throw new Error("no SEC filings matched the requested query, forms, and date range");
    firstCompany = { companyName: candidates[0].companyName, ticker: candidates[0].ticker, cik: candidates[0].cik };
    const reserveTranscriptSlot = sourceTypes.includes("earnings_transcripts") && maxDocuments > 1 ? 1 : 0;
    const secDocumentLimit = Math.max(1, maxDocuments - reserveTranscriptSlot);
    const maxSecScrapes = Math.min(candidates.length, secDocumentLimit);
    for (let i = 0; i < maxSecScrapes && rows.length < secDocumentLimit; i += 1) {
      rows.push(await extractSecRow(candidates[i], collectedAt));
    }
  }

  if (sourceTypes.includes("earnings_transcripts") && rows.length < maxDocuments) {
    const companyContext = firstCompany.companyName || firstCompany.ticker || firstCompany.cik ? firstCompany : { companyName: query, ticker: null, cik: null };
    const transcriptRows = await discoverTranscriptRows(companyContext, collectedAt, maxDocuments - rows.length);
    for (let i = 0; i < transcriptRows.length && rows.length < maxDocuments; i += 1) {
      rows.push(transcriptRows[i]);
    }
  }

  const out = outputMode === "grouped_by_company" ? groupedOutput(rows) : rows;
  process.stdout.write(JSON.stringify(out));
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});