scripts/seed-snap-negative.ts

Linked repository file from this build.

#!/usr/bin/env bun
import { businessKey, reviewKey } from "@/features/reviews/review-key";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { config } from "dotenv";
import { assertProductionConfirmed, resolveDatabaseTarget, errorDetails } from "./db/target";
import { isDeepStrictEqual } from "node:util";
import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg";

import { deriveFfyFromSampleMonth } from "@/features/reviews/downstream";
import {
  reviewSnapshots,
  reviewIndex,
  inboxEvents,
  inboxVisits,
  reviewTouches,
  snapPolicyTables,
} from "@/lib/db/schema";
import { reviewRows } from "@/lib/db/review-store";
import { SYNTHETIC_ATTESTATION } from "@/lib/import/snap-negative";
import { workflowLabels } from "@/features/reviews/types";
import type { SnapPolicy } from "@/programs/snap-negative/policy";
import {
  currentSampleMonth,
  generateSeedBatch,
  type SeedGeneratorOptions,
} from "@/programs/snap-negative/seed-generator";
import {
  DEMO_GENERATOR_SEED,
  DEMO_SAMPLE_MONTH,
  resetBaselineBatchOptions,
} from "@/programs/snap-negative/seed";

config({ quiet: true });

interface CliOptions {
  ffyFrom: number | null;
  ffyTo: number | null;
  sampleMonths: string[] | null;
  perMonth: number;
  seed: number;
  asOf?: Date;
  dryRun: boolean;
  fixtures: boolean;
  fixturesDir: string;
  skipExisting: boolean;
  reset: boolean;
  confirmProduction: boolean;
}

function usage(): string {
  return `Usage: bun run scripts/seed-snap-negative.ts [options]

Options:
  --month <YYYY-MM|current>  Sample month to generate (default: September 2026)
  --ffy <year>               Full federal fiscal year when --month is not set
  --ffy-from <year>          First FFY to generate (default: --ffy)
  --ffy-to <year>            Last FFY to generate (default: --ffy)
  --per-month <count>        Closed/suspended and rejected rows per month (default: 50)
  --seed <number>            Deterministic random seed (default: 42)
  --as-of <YYYY-MM-DD>       Workflow reference day (default: last sample-month day)
  --reset                    Clear all review snapshots before seeding (default for db:seed)
  --no-reset                 Keep existing snapshots; skip duplicates (default without --reset)
  --fixtures                 Write BIS fixture files alongside DB seeding
  --fixtures-dir <path>      Output directory for fixtures (default: data/seed/generated)
  --force                    Attempt all inserts; existing FFY + review numbers fail
  --dry-run                  Print summary without writing
  --confirm-production       Required to write to PROD_DATABASE_URL
  --help                     Show this help text

Target: QC_DB_TARGET=production uses PROD_DATABASE_URL; otherwise DATABASE_URL.
Both db:seed and db:seed:prod default to --reset. --dry-run never connects.
All writes run in one transaction. A failure leaves the database unchanged.
`;
}

function parseSampleMonth(value: string): string {
  if (value === "current") return currentSampleMonth();
  if (!/^\d{4}-(0[1-9]|1[0-2])$/.test(value)) {
    throw new Error(`Sample month must be YYYY-MM or "current"; received "${value}".`);
  }
  return value;
}

export function parseArgs(argv: string[]): CliOptions {
  const options: CliOptions = {
    ffyFrom: null,
    ffyTo: null,
    sampleMonths: null,
    perMonth: 50,
    seed: DEMO_GENERATOR_SEED,
    dryRun: false,
    fixtures: false,
    fixturesDir: join(process.cwd(), "data/seed/generated"),
    skipExisting: true,
    reset: false,
    confirmProduction: false,
  };

  const valuedOptions = new Set([
    "--month",
    "--ffy",
    "--ffy-from",
    "--ffy-to",
    "--per-month",
    "--seed",
    "--as-of",
    "--fixtures-dir",
  ]);
  for (let index = 0; index < argv.length; index++) {
    if (
      valuedOptions.has(argv[index]!) &&
      (!argv[index + 1] || argv[index + 1]!.startsWith("--"))
    ) {
      throw new Error(`${argv[index]} requires a value.`);
    }
    const arg = argv[index];
    switch (arg) {
      case "--help":
      case "-h":
        console.log(usage());
        process.exit(0);
      case "--month":
        options.sampleMonths = [parseSampleMonth(argv[++index] ?? "current")];
        break;
      case "--ffy": {
        const value = Number(argv[++index]);
        options.ffyFrom = value;
        options.ffyTo = value;
        break;
      }
      case "--ffy-from":
        options.ffyFrom = Number(argv[++index]);
        break;
      case "--ffy-to":
        options.ffyTo = Number(argv[++index]);
        break;
      case "--per-month":
        options.perMonth = Number(argv[++index]);
        break;
      case "--seed":
        options.seed = Number(argv[++index]);
        break;
      case "--as-of": {
        const value = argv[++index]!;
        const date = new Date(`${value}T12:00:00.000Z`);
        if (
          !/^\d{4}-\d{2}-\d{2}$/.test(value) ||
          !Number.isFinite(date.getTime()) ||
          date.toISOString().slice(0, 10) !== value
        )
          throw new Error("--as-of must be a valid YYYY-MM-DD date.");
        options.asOf = date;
        break;
      }
      case "--fixtures":
        options.fixtures = true;
        break;
      case "--fixtures-dir":
        options.fixturesDir = argv[++index] ?? options.fixturesDir;
        break;
      case "--dry-run":
        options.dryRun = true;
        break;
      case "--reset":
        options.reset = true;
        options.skipExisting = false;
        break;
      case "--confirm-production":
        options.confirmProduction = true;
        break;
      case "--no-reset":
        options.reset = false;
        options.skipExisting = true;
        break;
      case "--force":
        options.skipExisting = false;
        break;
      default:
        throw new Error(`Unknown argument: ${arg}\n\n${usage()}`);
    }
  }

  if (!options.sampleMonths && options.ffyFrom === null && options.ffyTo === null) {
    options.sampleMonths = [DEMO_SAMPLE_MONTH];
  }
  const anchor = deriveFfyFromSampleMonth(options.sampleMonths?.[0] ?? DEMO_SAMPLE_MONTH);
  options.ffyFrom ??= options.ffyTo ?? anchor;
  options.ffyTo ??= options.ffyFrom;
  if (
    ![options.ffyFrom, options.ffyTo].every(
      (year) => Number.isInteger(year) && year >= 1901 && year <= 9999,
    )
  ) {
    throw new Error("Fiscal years must be whole years from 1901 through 9999.");
  }
  if (!Number.isSafeInteger(options.seed)) throw new Error("--seed must be a safe integer.");

  if (options.ffyFrom > options.ffyTo) {
    throw new Error("--ffy-from must be less than or equal to --ffy-to.");
  }
  if (!Number.isInteger(options.perMonth) || options.perMonth < 1) {
    throw new Error("--per-month must be a positive integer.");
  }

  return options;
}

function writeFixtureFile(path: string, rows: string[]) {
  const content = [
    `# ${SYNTHETIC_ATTESTATION}`,
    "# Generated by scripts/seed-snap-negative.ts",
    ...rows,
    "",
  ].join("\n");
  writeFileSync(path, content, "utf8");
}

const SEED_CHUNK = 200;
type SeedDatabase = ReturnType<typeof drizzle>;

/** Write the batch in one transaction so a failure cannot leave a half-seeded database. */
async function writeSeed(
  db: SeedDatabase,
  batch: ReturnType<typeof generateSeedBatch>,
  existing: ReadonlySet<string>,
  policies: SnapPolicy[],
  reset: boolean,
) {
  const actor = { name: "Seed Generator", role: "program_manager" as const };
  const rows = batch.reviews
    .filter((review) => !existing.has(reviewKey(review)))
    .map((review) =>
      reviewRows({
        version: 0,
        requestId: `seed-${reviewKey(review)}-v0`,
        review,
        actor,
        savedAt: batch.asOf,
      }),
    );
  return db.transaction(async (tx) => {
    if (policies.length)
      await tx.insert(snapPolicyTables).values(
        policies.map((policy) => ({
          id: policy.id,
          effectiveFrom: policy.effectiveFrom,
          policy,
          publishedBy: "Seed Generator",
          publishedAt: batch.asOf,
        })),
      );
    let removed = 0;
    if (reset) {
      await tx.delete(inboxEvents);
      await tx.delete(inboxVisits);
      await tx.delete(reviewTouches);
      await tx.delete(reviewIndex);
      removed = (await tx.delete(reviewSnapshots).returning({ key: reviewSnapshots.reviewKey }))
        .length;
    }
    for (let start = 0; start < rows.length; start += SEED_CHUNK) {
      const chunk = rows.slice(start, start + SEED_CHUNK);
      await tx.insert(reviewSnapshots).values(chunk.map((row) => row.snapshot));
      await tx.insert(reviewIndex).values(chunk.map((row) => row.index));
      const events = chunk.flatMap((row) => row.events);
      if (events.length) await tx.insert(inboxEvents).values(events);
    }
    return { removed, inserted: rows.length, skipped: batch.reviews.length - rows.length };
  });
}

/** Refuse collisions before a reset. Published tables are never replaced by seeding. */
export function missingSeedPolicies(
  generated: readonly SnapPolicy[],
  existing: readonly SnapPolicy[],
) {
  return generated.filter((policy) => {
    const conflicts = existing.filter(
      (row) => row.id === policy.id || row.effectiveFrom === policy.effectiveFrom,
    );
    if (conflicts.some((row) => !isDeepStrictEqual(row, policy)))
      throw new Error(
        `Policy conflict for ${policy.effectiveFrom}. Seeding will not replace a published table.`,
      );
    return conflicts.length === 0;
  });
}

function summarizeBatch(
  batch: ReturnType<typeof generateSeedBatch>,
  options: SeedGeneratorOptions,
  sampleMonths: readonly string[],
) {
  const byState = new Map<string, number>();
  const byAction = new Map<string, number>();
  for (const review of batch.reviews) {
    const label = workflowLabels[review.state];
    byState.set(label, (byState.get(label) ?? 0) + 1);
    byAction.set(review.facts.negativeAction, (byAction.get(review.facts.negativeAction) ?? 0) + 1);
  }

  console.log(
    `Generated ${batch.reviews.length} reviews for sample month(s): ${sampleMonths.join(", ")}.`,
  );
  console.log(`Per month: ${options.perMonth} closed/suspended + ${options.perMonth} rejected.`);
  console.log("Workflow states:", Object.fromEntries(byState));
  console.log(`Reference date: ${batch.asOf}`);
  console.log(
    "Policy tables:",
    batch.policies.map((policy) => `${policy.effectiveFrom}: ${policy.id}`),
  );
  console.log("Negative actions:", Object.fromEntries(byAction));
  const errors = batch.reviews.filter((review) => review.outcome === "error");
  console.log(
    `Invalid findings: ${errors.length} reviews, ${errors.reduce((total, review) => total + review.facts.answers.variances.length, 0)} element/nature pairs.`,
  );
}

async function main() {
  const cli = parseArgs(process.argv.slice(2));
  const generatorOptions: SeedGeneratorOptions = resetBaselineBatchOptions({
    ffyFrom: cli.ffyFrom ?? undefined,
    ffyTo: cli.ffyTo ?? undefined,
    perMonth: cli.perMonth,
    seed: cli.seed ?? DEMO_GENERATOR_SEED,
    sampleMonths: cli.sampleMonths ?? undefined,
    preserveReviewNumbers: [],
    asOf: cli.asOf,
  });

  const batch = generateSeedBatch(generatorOptions);
  summarizeBatch(
    batch,
    generatorOptions,
    [...new Set(batch.reviews.map((review) => review.sampleMonth))].sort(),
  );

  if (cli.dryRun) {
    console.log("Dry run complete. No files or database records written.");
    return;
  }

  const target = resolveDatabaseTarget(process.env.QC_DB_TARGET);
  assertProductionConfirmed(target.target, cli.confirmProduction, "Seeding replaces its demo data");
  console.log(`Database: ${target.target}, ${target.label} (${target.variable})`);
  // A direct connection gives a real transaction on both local Postgres and Neon.
  const pool = new Pool({ connectionString: target.url });
  try {
    await seedTarget(drizzle(pool), batch, cli);
  } finally {
    await pool.end();
  }
}

async function seedTarget(
  db: SeedDatabase,
  batch: ReturnType<typeof generateSeedBatch>,
  cli: CliOptions,
) {
  const existingPolicies = await db
    .select({ policy: snapPolicyTables.policy })
    .from(snapPolicyTables);
  const policies = missingSeedPolicies(
    batch.policies,
    existingPolicies.map((row) => row.policy),
  );
  const existingKeys = new Set<string>();
  if (!cli.reset) {
    const existing = new Map(
      (
        await db
          .select({
            reviewNumber: reviewIndex.reviewNumber,
            program: reviewIndex.program,
            sampleMonth: reviewIndex.sampleMonth,
          })
          .from(reviewIndex)
      ).map((row) => [businessKey(row), row.sampleMonth]),
    );
    for (const review of batch.reviews) {
      const month = existing.get(businessKey(review));
      if (month && (month !== review.sampleMonth || !cli.skipExisting))
        throw new Error(
          `Review ${reviewKey(review)} already exists for ${month}. No records were changed. Use the same sample-month assignment or explicitly reset the synthetic dataset.`,
        );
      if (month) existingKeys.add(reviewKey(review));
    }
  }

  if (cli.fixtures) {
    mkdirSync(cli.fixturesDir, { recursive: true });
    writeFixtureFile(join(cli.fixturesDir, "closed-suspended.txt"), batch.closedSuspendedBisRows);
    writeFixtureFile(join(cli.fixturesDir, "rejected.txt"), batch.rejectedBisRows);
    writeFileSync(
      join(cli.fixturesDir, "policies.json"),
      `${JSON.stringify(batch.policies, null, 2)}\n`,
      "utf8",
    );
    console.log(`Wrote fixture files to ${cli.fixturesDir}.`);
  }

  const { removed, inserted, skipped } = await writeSeed(
    db,
    batch,
    existingKeys,
    policies,
    cli.reset,
  );
  if (cli.reset) console.log(`Reset complete. Removed ${removed} stored review snapshot(s).`);
  console.log(`Database seed complete. Inserted ${inserted}, skipped ${skipped}.`);
}

if (import.meta.main)
  main().catch((error) => {
    console.error(
      JSON.stringify(
        errorDetails(error, [process.env.DATABASE_URL ?? "", process.env.PROD_DATABASE_URL ?? ""]),
        null,
        2,
      ),
    );
    process.exit(1);
  });